2024-11-26 19:13:38,074 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-26 19:13:38,089 main DEBUG Took 0.012569 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-26 19:13:38,089 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-26 19:13:38,090 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-26 19:13:38,091 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-26 19:13:38,092 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 19:13:38,102 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-26 19:13:38,115 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 19:13:38,116 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 19:13:38,117 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 19:13:38,117 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 19:13:38,118 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 19:13:38,118 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 19:13:38,119 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 19:13:38,120 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 19:13:38,120 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 19:13:38,121 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 19:13:38,122 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 19:13:38,122 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 19:13:38,123 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 19:13:38,123 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 19:13:38,124 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 19:13:38,124 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 19:13:38,125 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 19:13:38,125 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 19:13:38,126 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 19:13:38,126 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 19:13:38,126 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 19:13:38,127 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 19:13:38,127 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 19:13:38,128 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 19:13:38,128 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 19:13:38,129 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-26 19:13:38,130 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 19:13:38,132 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-26 19:13:38,135 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-26 19:13:38,135 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-26 19:13:38,137 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-26 19:13:38,137 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-26 19:13:38,149 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-26 19:13:38,152 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-26 19:13:38,154 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-26 19:13:38,155 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-26 19:13:38,155 main DEBUG createAppenders(={Console}) 2024-11-26 19:13:38,156 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-11-26 19:13:38,157 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-26 19:13:38,157 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-11-26 19:13:38,158 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-26 19:13:38,158 main DEBUG OutputStream closed 2024-11-26 19:13:38,159 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-26 19:13:38,159 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-26 19:13:38,159 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-11-26 19:13:38,241 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-26 19:13:38,243 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-26 19:13:38,244 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-26 19:13:38,245 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-26 19:13:38,245 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-26 19:13:38,246 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-26 19:13:38,246 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-26 19:13:38,246 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-26 19:13:38,246 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-26 19:13:38,247 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-26 19:13:38,247 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-26 19:13:38,247 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-26 19:13:38,248 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-26 19:13:38,248 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-26 19:13:38,248 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-26 19:13:38,248 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-26 19:13:38,249 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-26 19:13:38,249 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-26 19:13:38,252 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-26 19:13:38,252 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-11-26 19:13:38,252 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-26 19:13:38,253 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-11-26T19:13:38,490 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/efb451b3-ba8c-40d2-3b0c-db6f88deea36 2024-11-26 19:13:38,494 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-26 19:13:38,494 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-26T19:13:38,503 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithEagerPolicy timeout: 13 mins 2024-11-26T19:13:38,521 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-26T19:13:38,540 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/efb451b3-ba8c-40d2-3b0c-db6f88deea36/cluster_7d054ae1-c0e9-960d-2aea-ddd34a033944, deleteOnExit=true 2024-11-26T19:13:38,541 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-26T19:13:38,542 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/efb451b3-ba8c-40d2-3b0c-db6f88deea36/test.cache.data in system properties and HBase conf 2024-11-26T19:13:38,542 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/efb451b3-ba8c-40d2-3b0c-db6f88deea36/hadoop.tmp.dir in system properties and HBase conf 2024-11-26T19:13:38,543 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/efb451b3-ba8c-40d2-3b0c-db6f88deea36/hadoop.log.dir in system properties and HBase conf 2024-11-26T19:13:38,544 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/efb451b3-ba8c-40d2-3b0c-db6f88deea36/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-26T19:13:38,544 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/efb451b3-ba8c-40d2-3b0c-db6f88deea36/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-26T19:13:38,545 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-26T19:13:38,644 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-26T19:13:38,749 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-26T19:13:38,753 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/efb451b3-ba8c-40d2-3b0c-db6f88deea36/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-26T19:13:38,754 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/efb451b3-ba8c-40d2-3b0c-db6f88deea36/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-26T19:13:38,754 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/efb451b3-ba8c-40d2-3b0c-db6f88deea36/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-26T19:13:38,755 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/efb451b3-ba8c-40d2-3b0c-db6f88deea36/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-26T19:13:38,756 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/efb451b3-ba8c-40d2-3b0c-db6f88deea36/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-26T19:13:38,756 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/efb451b3-ba8c-40d2-3b0c-db6f88deea36/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-26T19:13:38,757 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/efb451b3-ba8c-40d2-3b0c-db6f88deea36/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-26T19:13:38,757 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/efb451b3-ba8c-40d2-3b0c-db6f88deea36/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-26T19:13:38,758 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/efb451b3-ba8c-40d2-3b0c-db6f88deea36/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-26T19:13:38,758 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/efb451b3-ba8c-40d2-3b0c-db6f88deea36/nfs.dump.dir in system properties and HBase conf 2024-11-26T19:13:38,759 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/efb451b3-ba8c-40d2-3b0c-db6f88deea36/java.io.tmpdir in system properties and HBase conf 2024-11-26T19:13:38,760 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/efb451b3-ba8c-40d2-3b0c-db6f88deea36/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-26T19:13:38,760 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/efb451b3-ba8c-40d2-3b0c-db6f88deea36/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-26T19:13:38,761 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/efb451b3-ba8c-40d2-3b0c-db6f88deea36/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-26T19:13:39,570 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-26T19:13:39,651 INFO [Time-limited test {}] log.Log(170): Logging initialized @2270ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-26T19:13:39,728 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-26T19:13:39,792 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-26T19:13:39,813 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-26T19:13:39,814 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-26T19:13:39,815 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-26T19:13:39,827 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-26T19:13:39,830 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1a58fd8a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/efb451b3-ba8c-40d2-3b0c-db6f88deea36/hadoop.log.dir/,AVAILABLE} 2024-11-26T19:13:39,831 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@130d168e{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-26T19:13:40,032 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@2341a969{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/efb451b3-ba8c-40d2-3b0c-db6f88deea36/java.io.tmpdir/jetty-localhost-44171-hadoop-hdfs-3_4_1-tests_jar-_-any-768430229094595239/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-26T19:13:40,039 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2d064751{HTTP/1.1, (http/1.1)}{localhost:44171} 2024-11-26T19:13:40,039 INFO [Time-limited test {}] server.Server(415): Started @2659ms 2024-11-26T19:13:40,428 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-26T19:13:40,435 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-26T19:13:40,436 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-26T19:13:40,436 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-26T19:13:40,437 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-26T19:13:40,437 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@54a7b62e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/efb451b3-ba8c-40d2-3b0c-db6f88deea36/hadoop.log.dir/,AVAILABLE} 2024-11-26T19:13:40,438 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@72782cb0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-26T19:13:40,589 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@292d50c4{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/efb451b3-ba8c-40d2-3b0c-db6f88deea36/java.io.tmpdir/jetty-localhost-33189-hadoop-hdfs-3_4_1-tests_jar-_-any-4301778365130224834/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-26T19:13:40,591 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@667c6eba{HTTP/1.1, (http/1.1)}{localhost:33189} 2024-11-26T19:13:40,591 INFO [Time-limited test {}] server.Server(415): Started @3211ms 2024-11-26T19:13:40,660 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-26T19:13:41,959 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/efb451b3-ba8c-40d2-3b0c-db6f88deea36/cluster_7d054ae1-c0e9-960d-2aea-ddd34a033944/data/data2/current/BP-777500891-172.17.0.2-1732648419325/current, will proceed with Du for space computation calculation, 2024-11-26T19:13:41,959 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/efb451b3-ba8c-40d2-3b0c-db6f88deea36/cluster_7d054ae1-c0e9-960d-2aea-ddd34a033944/data/data1/current/BP-777500891-172.17.0.2-1732648419325/current, will proceed with Du for space computation calculation, 2024-11-26T19:13:42,002 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-26T19:13:42,065 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xd48c5fe8977a4a74 with lease ID 0x654cdd2b8ec438f0: Processing first storage report for DS-754256f3-2cdd-43bf-a093-a98dd3bb7358 from datanode DatanodeRegistration(127.0.0.1:34683, datanodeUuid=4ad75c6e-6c93-4de8-8725-1def329addcb, infoPort=38333, infoSecurePort=0, ipcPort=44441, storageInfo=lv=-57;cid=testClusterID;nsid=1485412032;c=1732648419325) 2024-11-26T19:13:42,067 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xd48c5fe8977a4a74 with lease ID 0x654cdd2b8ec438f0: from storage DS-754256f3-2cdd-43bf-a093-a98dd3bb7358 node DatanodeRegistration(127.0.0.1:34683, datanodeUuid=4ad75c6e-6c93-4de8-8725-1def329addcb, infoPort=38333, infoSecurePort=0, ipcPort=44441, storageInfo=lv=-57;cid=testClusterID;nsid=1485412032;c=1732648419325), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-11-26T19:13:42,067 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xd48c5fe8977a4a74 with lease ID 0x654cdd2b8ec438f0: Processing first storage report for DS-f04740af-bdb1-4c3a-a676-bb99801e1e87 from datanode DatanodeRegistration(127.0.0.1:34683, datanodeUuid=4ad75c6e-6c93-4de8-8725-1def329addcb, infoPort=38333, infoSecurePort=0, ipcPort=44441, storageInfo=lv=-57;cid=testClusterID;nsid=1485412032;c=1732648419325) 2024-11-26T19:13:42,068 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xd48c5fe8977a4a74 with lease ID 0x654cdd2b8ec438f0: from storage DS-f04740af-bdb1-4c3a-a676-bb99801e1e87 node DatanodeRegistration(127.0.0.1:34683, datanodeUuid=4ad75c6e-6c93-4de8-8725-1def329addcb, infoPort=38333, infoSecurePort=0, ipcPort=44441, storageInfo=lv=-57;cid=testClusterID;nsid=1485412032;c=1732648419325), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-26T19:13:42,139 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/efb451b3-ba8c-40d2-3b0c-db6f88deea36 2024-11-26T19:13:42,238 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/efb451b3-ba8c-40d2-3b0c-db6f88deea36/cluster_7d054ae1-c0e9-960d-2aea-ddd34a033944/zookeeper_0, clientPort=62039, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/efb451b3-ba8c-40d2-3b0c-db6f88deea36/cluster_7d054ae1-c0e9-960d-2aea-ddd34a033944/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/efb451b3-ba8c-40d2-3b0c-db6f88deea36/cluster_7d054ae1-c0e9-960d-2aea-ddd34a033944/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-26T19:13:42,252 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=62039 2024-11-26T19:13:42,264 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T19:13:42,268 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T19:13:42,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741825_1001 (size=7) 2024-11-26T19:13:42,928 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16 with version=8 2024-11-26T19:13:42,929 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/hbase-staging 2024-11-26T19:13:43,045 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-26T19:13:43,297 INFO [Time-limited test {}] client.ConnectionUtils(128): master/fe76acf732b2:0 server-side Connection retries=45 2024-11-26T19:13:43,308 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T19:13:43,309 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-26T19:13:43,313 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-26T19:13:43,313 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T19:13:43,314 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-26T19:13:43,463 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-26T19:13:43,537 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-26T19:13:43,550 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-26T19:13:43,556 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-26T19:13:43,584 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 3068 (auto-detected) 2024-11-26T19:13:43,585 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-26T19:13:43,610 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:33811 2024-11-26T19:13:43,643 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:33811 connecting to ZooKeeper ensemble=127.0.0.1:62039 2024-11-26T19:13:43,689 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:338110x0, quorum=127.0.0.1:62039, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-26T19:13:43,694 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:33811-0x10155b2b5d30000 connected 2024-11-26T19:13:43,755 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T19:13:43,759 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T19:13:43,773 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:33811-0x10155b2b5d30000, quorum=127.0.0.1:62039, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-26T19:13:43,778 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16, hbase.cluster.distributed=false 2024-11-26T19:13:43,812 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:33811-0x10155b2b5d30000, quorum=127.0.0.1:62039, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-26T19:13:43,832 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33811 2024-11-26T19:13:43,833 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33811 2024-11-26T19:13:43,844 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33811 2024-11-26T19:13:43,864 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33811 2024-11-26T19:13:43,865 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33811 2024-11-26T19:13:44,027 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/fe76acf732b2:0 server-side Connection retries=45 2024-11-26T19:13:44,030 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T19:13:44,030 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-26T19:13:44,031 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-26T19:13:44,031 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T19:13:44,031 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-26T19:13:44,035 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-26T19:13:44,038 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-26T19:13:44,041 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:42785 2024-11-26T19:13:44,043 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:42785 connecting to ZooKeeper ensemble=127.0.0.1:62039 2024-11-26T19:13:44,045 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T19:13:44,049 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T19:13:44,060 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:427850x0, quorum=127.0.0.1:62039, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-26T19:13:44,063 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:427850x0, quorum=127.0.0.1:62039, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-26T19:13:44,065 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:42785-0x10155b2b5d30001 connected 2024-11-26T19:13:44,069 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-26T19:13:44,080 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-26T19:13:44,083 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42785-0x10155b2b5d30001, quorum=127.0.0.1:62039, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-26T19:13:44,091 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42785-0x10155b2b5d30001, quorum=127.0.0.1:62039, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-26T19:13:44,092 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42785 2024-11-26T19:13:44,093 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42785 2024-11-26T19:13:44,096 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42785 2024-11-26T19:13:44,100 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42785 2024-11-26T19:13:44,101 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42785 2024-11-26T19:13:44,117 DEBUG [M:0;fe76acf732b2:33811 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;fe76acf732b2:33811 2024-11-26T19:13:44,118 INFO [master/fe76acf732b2:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/fe76acf732b2,33811,1732648423099 2024-11-26T19:13:44,128 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33811-0x10155b2b5d30000, quorum=127.0.0.1:62039, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-26T19:13:44,128 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42785-0x10155b2b5d30001, quorum=127.0.0.1:62039, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-26T19:13:44,129 DEBUG [master/fe76acf732b2:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:33811-0x10155b2b5d30000, quorum=127.0.0.1:62039, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/fe76acf732b2,33811,1732648423099 2024-11-26T19:13:44,155 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42785-0x10155b2b5d30001, quorum=127.0.0.1:62039, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-26T19:13:44,155 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33811-0x10155b2b5d30000, quorum=127.0.0.1:62039, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T19:13:44,155 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42785-0x10155b2b5d30001, quorum=127.0.0.1:62039, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T19:13:44,156 DEBUG [master/fe76acf732b2:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:33811-0x10155b2b5d30000, quorum=127.0.0.1:62039, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-26T19:13:44,159 INFO [master/fe76acf732b2:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/fe76acf732b2,33811,1732648423099 from backup master directory 2024-11-26T19:13:44,163 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33811-0x10155b2b5d30000, quorum=127.0.0.1:62039, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/fe76acf732b2,33811,1732648423099 2024-11-26T19:13:44,163 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42785-0x10155b2b5d30001, quorum=127.0.0.1:62039, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-26T19:13:44,163 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33811-0x10155b2b5d30000, quorum=127.0.0.1:62039, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-26T19:13:44,165 WARN [master/fe76acf732b2:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-26T19:13:44,165 INFO [master/fe76acf732b2:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=fe76acf732b2,33811,1732648423099 2024-11-26T19:13:44,168 INFO [master/fe76acf732b2:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-26T19:13:44,173 INFO [master/fe76acf732b2:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-26T19:13:44,234 DEBUG [master/fe76acf732b2:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/hbase.id] with ID: 3cec4e24-71db-4cd0-b63a-4ea30a9b160b 2024-11-26T19:13:44,234 DEBUG [master/fe76acf732b2:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/.tmp/hbase.id 2024-11-26T19:13:44,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741826_1002 (size=42) 2024-11-26T19:13:44,647 DEBUG [master/fe76acf732b2:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/.tmp/hbase.id]:[hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/hbase.id] 2024-11-26T19:13:44,703 INFO [master/fe76acf732b2:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T19:13:44,709 INFO [master/fe76acf732b2:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-26T19:13:44,737 INFO [master/fe76acf732b2:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 25ms. 2024-11-26T19:13:44,744 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33811-0x10155b2b5d30000, quorum=127.0.0.1:62039, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T19:13:44,744 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42785-0x10155b2b5d30001, quorum=127.0.0.1:62039, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T19:13:44,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741827_1003 (size=196) 2024-11-26T19:13:45,184 INFO [master/fe76acf732b2:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-26T19:13:45,188 INFO [master/fe76acf732b2:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-26T19:13:45,208 DEBUG [master/fe76acf732b2:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:13:45,214 INFO [master/fe76acf732b2:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T19:13:45,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741828_1004 (size=1189) 2024-11-26T19:13:45,663 INFO [master/fe76acf732b2:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/MasterData/data/master/store 2024-11-26T19:13:45,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741829_1005 (size=34) 2024-11-26T19:13:45,686 INFO [master/fe76acf732b2:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-26T19:13:45,689 DEBUG [master/fe76acf732b2:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T19:13:45,690 DEBUG [master/fe76acf732b2:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-26T19:13:45,691 INFO [master/fe76acf732b2:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T19:13:45,691 DEBUG [master/fe76acf732b2:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T19:13:45,692 DEBUG [master/fe76acf732b2:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-26T19:13:45,693 DEBUG [master/fe76acf732b2:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T19:13:45,693 INFO [master/fe76acf732b2:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T19:13:45,694 DEBUG [master/fe76acf732b2:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732648425690Disabling compacts and flushes for region at 1732648425690Disabling writes for close at 1732648425692 (+2 ms)Writing region close event to WAL at 1732648425693 (+1 ms)Closed at 1732648425693 2024-11-26T19:13:45,696 WARN [master/fe76acf732b2:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/MasterData/data/master/store/.initializing 2024-11-26T19:13:45,696 DEBUG [master/fe76acf732b2:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/MasterData/WALs/fe76acf732b2,33811,1732648423099 2024-11-26T19:13:45,705 INFO [master/fe76acf732b2:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-26T19:13:45,720 INFO [master/fe76acf732b2:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=fe76acf732b2%2C33811%2C1732648423099, suffix=, logDir=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/MasterData/WALs/fe76acf732b2,33811,1732648423099, archiveDir=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/MasterData/oldWALs, maxLogs=10 2024-11-26T19:13:45,743 DEBUG [master/fe76acf732b2:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/MasterData/WALs/fe76acf732b2,33811,1732648423099/fe76acf732b2%2C33811%2C1732648423099.1732648425725, exclude list is [], retry=0 2024-11-26T19:13:45,762 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34683,DS-754256f3-2cdd-43bf-a093-a98dd3bb7358,DISK] 2024-11-26T19:13:45,765 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-26T19:13:45,800 INFO [master/fe76acf732b2:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/MasterData/WALs/fe76acf732b2,33811,1732648423099/fe76acf732b2%2C33811%2C1732648423099.1732648425725 2024-11-26T19:13:45,801 DEBUG [master/fe76acf732b2:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38333:38333)] 2024-11-26T19:13:45,801 DEBUG [master/fe76acf732b2:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-26T19:13:45,802 DEBUG [master/fe76acf732b2:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T19:13:45,805 DEBUG [master/fe76acf732b2:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T19:13:45,806 DEBUG [master/fe76acf732b2:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T19:13:45,843 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-26T19:13:45,868 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-26T19:13:45,872 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:13:45,875 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T19:13:45,875 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-26T19:13:45,878 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-26T19:13:45,878 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:13:45,879 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T19:13:45,880 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-26T19:13:45,882 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-26T19:13:45,882 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:13:45,883 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T19:13:45,883 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-26T19:13:45,886 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-26T19:13:45,886 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:13:45,887 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T19:13:45,887 DEBUG [master/fe76acf732b2:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T19:13:45,891 DEBUG [master/fe76acf732b2:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-26T19:13:45,892 DEBUG [master/fe76acf732b2:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-26T19:13:45,897 DEBUG [master/fe76acf732b2:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T19:13:45,897 DEBUG [master/fe76acf732b2:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T19:13:45,901 DEBUG [master/fe76acf732b2:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-26T19:13:45,904 DEBUG [master/fe76acf732b2:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T19:13:45,908 DEBUG [master/fe76acf732b2:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T19:13:45,909 INFO [master/fe76acf732b2:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65486677, jitterRate=-0.024172469973564148}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-26T19:13:45,914 DEBUG [master/fe76acf732b2:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732648425819Initializing all the Stores at 1732648425821 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732648425821Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732648425822 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732648425822Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732648425822Cleaning up temporary data from old regions at 1732648425897 (+75 ms)Region opened successfully at 1732648425914 (+17 ms) 2024-11-26T19:13:45,915 INFO [master/fe76acf732b2:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-26T19:13:45,948 DEBUG [master/fe76acf732b2:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46394c14, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=fe76acf732b2/172.17.0.2:0 2024-11-26T19:13:45,981 INFO [master/fe76acf732b2:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-26T19:13:45,992 INFO [master/fe76acf732b2:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-26T19:13:45,992 INFO [master/fe76acf732b2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-26T19:13:45,995 INFO [master/fe76acf732b2:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-26T19:13:45,996 INFO [master/fe76acf732b2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-26T19:13:46,001 INFO [master/fe76acf732b2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-26T19:13:46,001 INFO [master/fe76acf732b2:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-26T19:13:46,026 INFO [master/fe76acf732b2:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-26T19:13:46,036 DEBUG [master/fe76acf732b2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33811-0x10155b2b5d30000, quorum=127.0.0.1:62039, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-26T19:13:46,038 DEBUG [master/fe76acf732b2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-26T19:13:46,042 INFO [master/fe76acf732b2:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-26T19:13:46,044 DEBUG [master/fe76acf732b2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33811-0x10155b2b5d30000, quorum=127.0.0.1:62039, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-26T19:13:46,045 DEBUG [master/fe76acf732b2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-26T19:13:46,048 INFO [master/fe76acf732b2:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-26T19:13:46,051 DEBUG [master/fe76acf732b2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33811-0x10155b2b5d30000, quorum=127.0.0.1:62039, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-26T19:13:46,053 DEBUG [master/fe76acf732b2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-26T19:13:46,055 DEBUG [master/fe76acf732b2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33811-0x10155b2b5d30000, quorum=127.0.0.1:62039, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-26T19:13:46,057 DEBUG [master/fe76acf732b2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-26T19:13:46,078 DEBUG [master/fe76acf732b2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33811-0x10155b2b5d30000, quorum=127.0.0.1:62039, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-26T19:13:46,079 DEBUG [master/fe76acf732b2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-26T19:13:46,084 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33811-0x10155b2b5d30000, quorum=127.0.0.1:62039, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-26T19:13:46,084 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42785-0x10155b2b5d30001, quorum=127.0.0.1:62039, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-26T19:13:46,085 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33811-0x10155b2b5d30000, quorum=127.0.0.1:62039, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T19:13:46,085 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42785-0x10155b2b5d30001, quorum=127.0.0.1:62039, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T19:13:46,087 INFO [master/fe76acf732b2:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=fe76acf732b2,33811,1732648423099, sessionid=0x10155b2b5d30000, setting cluster-up flag (Was=false) 2024-11-26T19:13:46,103 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42785-0x10155b2b5d30001, quorum=127.0.0.1:62039, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T19:13:46,103 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33811-0x10155b2b5d30000, quorum=127.0.0.1:62039, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T19:13:46,110 DEBUG [master/fe76acf732b2:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-26T19:13:46,112 DEBUG [master/fe76acf732b2:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=fe76acf732b2,33811,1732648423099 2024-11-26T19:13:46,120 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33811-0x10155b2b5d30000, quorum=127.0.0.1:62039, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T19:13:46,120 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42785-0x10155b2b5d30001, quorum=127.0.0.1:62039, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T19:13:46,127 DEBUG [master/fe76acf732b2:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-26T19:13:46,129 DEBUG [master/fe76acf732b2:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=fe76acf732b2,33811,1732648423099 2024-11-26T19:13:46,137 INFO [master/fe76acf732b2:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-26T19:13:46,210 INFO [RS:0;fe76acf732b2:42785 {}] regionserver.HRegionServer(746): ClusterId : 3cec4e24-71db-4cd0-b63a-4ea30a9b160b 2024-11-26T19:13:46,214 DEBUG [RS:0;fe76acf732b2:42785 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-26T19:13:46,219 DEBUG [RS:0;fe76acf732b2:42785 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-26T19:13:46,220 DEBUG [RS:0;fe76acf732b2:42785 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-26T19:13:46,223 DEBUG [RS:0;fe76acf732b2:42785 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-26T19:13:46,224 DEBUG [RS:0;fe76acf732b2:42785 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52444bb8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=fe76acf732b2/172.17.0.2:0 2024-11-26T19:13:46,225 DEBUG [master/fe76acf732b2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-26T19:13:46,237 INFO [master/fe76acf732b2:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-26T19:13:46,240 DEBUG [RS:0;fe76acf732b2:42785 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;fe76acf732b2:42785 2024-11-26T19:13:46,245 INFO [RS:0;fe76acf732b2:42785 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-26T19:13:46,245 INFO [master/fe76acf732b2:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-26T19:13:46,245 INFO [RS:0;fe76acf732b2:42785 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-26T19:13:46,245 DEBUG [RS:0;fe76acf732b2:42785 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-26T19:13:46,249 INFO [RS:0;fe76acf732b2:42785 {}] regionserver.HRegionServer(2659): reportForDuty to master=fe76acf732b2,33811,1732648423099 with port=42785, startcode=1732648423970 2024-11-26T19:13:46,252 DEBUG [master/fe76acf732b2:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: fe76acf732b2,33811,1732648423099 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-26T19:13:46,260 DEBUG [master/fe76acf732b2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/fe76acf732b2:0, corePoolSize=5, maxPoolSize=5 2024-11-26T19:13:46,261 DEBUG [master/fe76acf732b2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/fe76acf732b2:0, corePoolSize=5, maxPoolSize=5 2024-11-26T19:13:46,261 DEBUG [master/fe76acf732b2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/fe76acf732b2:0, corePoolSize=5, maxPoolSize=5 2024-11-26T19:13:46,261 DEBUG [master/fe76acf732b2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/fe76acf732b2:0, corePoolSize=5, maxPoolSize=5 2024-11-26T19:13:46,261 DEBUG [master/fe76acf732b2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/fe76acf732b2:0, corePoolSize=10, maxPoolSize=10 2024-11-26T19:13:46,262 DEBUG [master/fe76acf732b2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/fe76acf732b2:0, corePoolSize=1, maxPoolSize=1 2024-11-26T19:13:46,262 DEBUG [master/fe76acf732b2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/fe76acf732b2:0, corePoolSize=2, maxPoolSize=2 2024-11-26T19:13:46,262 DEBUG [master/fe76acf732b2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/fe76acf732b2:0, corePoolSize=1, maxPoolSize=1 2024-11-26T19:13:46,265 DEBUG [RS:0;fe76acf732b2:42785 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-26T19:13:46,276 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-26T19:13:46,277 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-26T19:13:46,282 INFO [master/fe76acf732b2:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732648456282 2024-11-26T19:13:46,283 INFO [master/fe76acf732b2:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-26T19:13:46,284 INFO [master/fe76acf732b2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-26T19:13:46,285 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:13:46,285 INFO [PEWorker-2 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-26T19:13:46,288 INFO [master/fe76acf732b2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-26T19:13:46,289 INFO [master/fe76acf732b2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-26T19:13:46,289 INFO [master/fe76acf732b2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-26T19:13:46,289 INFO [master/fe76acf732b2:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-26T19:13:46,293 INFO [master/fe76acf732b2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-26T19:13:46,301 INFO [master/fe76acf732b2:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-26T19:13:46,305 INFO [master/fe76acf732b2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-26T19:13:46,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741831_1007 (size=1321) 2024-11-26T19:13:46,305 INFO [master/fe76acf732b2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-26T19:13:46,308 INFO [master/fe76acf732b2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-26T19:13:46,309 INFO [master/fe76acf732b2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-26T19:13:46,311 DEBUG [master/fe76acf732b2:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/fe76acf732b2:0:becomeActiveMaster-HFileCleaner.large.0-1732648426310,5,FailOnTimeoutGroup] 2024-11-26T19:13:46,314 DEBUG [master/fe76acf732b2:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/fe76acf732b2:0:becomeActiveMaster-HFileCleaner.small.0-1732648426311,5,FailOnTimeoutGroup] 2024-11-26T19:13:46,314 INFO [master/fe76acf732b2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-26T19:13:46,315 INFO [master/fe76acf732b2:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-26T19:13:46,317 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53971, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-26T19:13:46,320 INFO [master/fe76acf732b2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-26T19:13:46,321 INFO [master/fe76acf732b2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-26T19:13:46,323 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33811 {}] master.ServerManager(363): Checking decommissioned status of RegionServer fe76acf732b2,42785,1732648423970 2024-11-26T19:13:46,326 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33811 {}] master.ServerManager(517): Registering regionserver=fe76acf732b2,42785,1732648423970 2024-11-26T19:13:46,343 DEBUG [RS:0;fe76acf732b2:42785 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16 2024-11-26T19:13:46,343 DEBUG [RS:0;fe76acf732b2:42785 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:43653 2024-11-26T19:13:46,344 DEBUG [RS:0;fe76acf732b2:42785 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-26T19:13:46,349 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33811-0x10155b2b5d30000, quorum=127.0.0.1:62039, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-26T19:13:46,349 DEBUG [RS:0;fe76acf732b2:42785 {}] zookeeper.ZKUtil(111): regionserver:42785-0x10155b2b5d30001, quorum=127.0.0.1:62039, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/fe76acf732b2,42785,1732648423970 2024-11-26T19:13:46,350 WARN [RS:0;fe76acf732b2:42785 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-26T19:13:46,350 INFO [RS:0;fe76acf732b2:42785 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T19:13:46,350 DEBUG [RS:0;fe76acf732b2:42785 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/WALs/fe76acf732b2,42785,1732648423970 2024-11-26T19:13:46,353 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [fe76acf732b2,42785,1732648423970] 2024-11-26T19:13:46,379 INFO [RS:0;fe76acf732b2:42785 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-26T19:13:46,393 INFO [RS:0;fe76acf732b2:42785 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-26T19:13:46,401 INFO [RS:0;fe76acf732b2:42785 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-26T19:13:46,401 INFO [RS:0;fe76acf732b2:42785 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T19:13:46,403 INFO [RS:0;fe76acf732b2:42785 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-26T19:13:46,410 INFO [RS:0;fe76acf732b2:42785 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-26T19:13:46,411 INFO [RS:0;fe76acf732b2:42785 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-26T19:13:46,412 DEBUG [RS:0;fe76acf732b2:42785 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/fe76acf732b2:0, corePoolSize=1, maxPoolSize=1 2024-11-26T19:13:46,412 DEBUG [RS:0;fe76acf732b2:42785 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/fe76acf732b2:0, corePoolSize=1, maxPoolSize=1 2024-11-26T19:13:46,412 DEBUG [RS:0;fe76acf732b2:42785 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/fe76acf732b2:0, corePoolSize=1, maxPoolSize=1 2024-11-26T19:13:46,413 DEBUG [RS:0;fe76acf732b2:42785 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/fe76acf732b2:0, corePoolSize=1, maxPoolSize=1 2024-11-26T19:13:46,413 DEBUG [RS:0;fe76acf732b2:42785 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/fe76acf732b2:0, corePoolSize=1, maxPoolSize=1 2024-11-26T19:13:46,413 DEBUG [RS:0;fe76acf732b2:42785 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/fe76acf732b2:0, corePoolSize=2, maxPoolSize=2 2024-11-26T19:13:46,413 DEBUG [RS:0;fe76acf732b2:42785 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/fe76acf732b2:0, corePoolSize=1, maxPoolSize=1 2024-11-26T19:13:46,414 DEBUG [RS:0;fe76acf732b2:42785 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/fe76acf732b2:0, corePoolSize=1, maxPoolSize=1 2024-11-26T19:13:46,414 DEBUG [RS:0;fe76acf732b2:42785 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/fe76acf732b2:0, corePoolSize=1, maxPoolSize=1 2024-11-26T19:13:46,414 DEBUG [RS:0;fe76acf732b2:42785 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/fe76acf732b2:0, corePoolSize=1, maxPoolSize=1 2024-11-26T19:13:46,414 DEBUG [RS:0;fe76acf732b2:42785 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/fe76acf732b2:0, corePoolSize=1, maxPoolSize=1 2024-11-26T19:13:46,414 DEBUG [RS:0;fe76acf732b2:42785 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/fe76acf732b2:0, corePoolSize=1, maxPoolSize=1 2024-11-26T19:13:46,414 DEBUG [RS:0;fe76acf732b2:42785 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/fe76acf732b2:0, corePoolSize=3, maxPoolSize=3 2024-11-26T19:13:46,415 DEBUG [RS:0;fe76acf732b2:42785 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0, corePoolSize=3, maxPoolSize=3 2024-11-26T19:13:46,416 INFO [RS:0;fe76acf732b2:42785 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-26T19:13:46,416 INFO [RS:0;fe76acf732b2:42785 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-26T19:13:46,416 INFO [RS:0;fe76acf732b2:42785 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T19:13:46,417 INFO [RS:0;fe76acf732b2:42785 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-26T19:13:46,417 INFO [RS:0;fe76acf732b2:42785 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-26T19:13:46,417 INFO [RS:0;fe76acf732b2:42785 {}] hbase.ChoreService(168): Chore ScheduledChore name=fe76acf732b2,42785,1732648423970-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-26T19:13:46,442 INFO [RS:0;fe76acf732b2:42785 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-26T19:13:46,444 INFO [RS:0;fe76acf732b2:42785 {}] hbase.ChoreService(168): Chore ScheduledChore name=fe76acf732b2,42785,1732648423970-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T19:13:46,445 INFO [RS:0;fe76acf732b2:42785 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T19:13:46,445 INFO [RS:0;fe76acf732b2:42785 {}] regionserver.Replication(171): fe76acf732b2,42785,1732648423970 started 2024-11-26T19:13:46,469 INFO [RS:0;fe76acf732b2:42785 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T19:13:46,469 INFO [RS:0;fe76acf732b2:42785 {}] regionserver.HRegionServer(1482): Serving as fe76acf732b2,42785,1732648423970, RpcServer on fe76acf732b2/172.17.0.2:42785, sessionid=0x10155b2b5d30001 2024-11-26T19:13:46,470 DEBUG [RS:0;fe76acf732b2:42785 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-26T19:13:46,470 DEBUG [RS:0;fe76acf732b2:42785 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager fe76acf732b2,42785,1732648423970 2024-11-26T19:13:46,471 DEBUG [RS:0;fe76acf732b2:42785 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'fe76acf732b2,42785,1732648423970' 2024-11-26T19:13:46,471 DEBUG [RS:0;fe76acf732b2:42785 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-26T19:13:46,472 DEBUG [RS:0;fe76acf732b2:42785 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-26T19:13:46,473 DEBUG [RS:0;fe76acf732b2:42785 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-26T19:13:46,473 DEBUG [RS:0;fe76acf732b2:42785 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-26T19:13:46,473 DEBUG [RS:0;fe76acf732b2:42785 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager fe76acf732b2,42785,1732648423970 2024-11-26T19:13:46,473 DEBUG [RS:0;fe76acf732b2:42785 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'fe76acf732b2,42785,1732648423970' 2024-11-26T19:13:46,474 DEBUG [RS:0;fe76acf732b2:42785 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-26T19:13:46,474 DEBUG [RS:0;fe76acf732b2:42785 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-26T19:13:46,475 DEBUG [RS:0;fe76acf732b2:42785 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-26T19:13:46,475 INFO [RS:0;fe76acf732b2:42785 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-26T19:13:46,476 INFO [RS:0;fe76acf732b2:42785 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-26T19:13:46,581 INFO [RS:0;fe76acf732b2:42785 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-26T19:13:46,585 INFO [RS:0;fe76acf732b2:42785 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=fe76acf732b2%2C42785%2C1732648423970, suffix=, logDir=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/WALs/fe76acf732b2,42785,1732648423970, archiveDir=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/oldWALs, maxLogs=32 2024-11-26T19:13:46,602 DEBUG [RS:0;fe76acf732b2:42785 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/WALs/fe76acf732b2,42785,1732648423970/fe76acf732b2%2C42785%2C1732648423970.1732648426587, exclude list is [], retry=0 2024-11-26T19:13:46,606 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34683,DS-754256f3-2cdd-43bf-a093-a98dd3bb7358,DISK] 2024-11-26T19:13:46,609 INFO [RS:0;fe76acf732b2:42785 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/WALs/fe76acf732b2,42785,1732648423970/fe76acf732b2%2C42785%2C1732648423970.1732648426587 2024-11-26T19:13:46,610 DEBUG [RS:0;fe76acf732b2:42785 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38333:38333)] 2024-11-26T19:13:46,708 INFO [PEWorker-2 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-26T19:13:46,708 INFO [PEWorker-2 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16 2024-11-26T19:13:46,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741833_1009 (size=32) 2024-11-26T19:13:47,120 DEBUG [PEWorker-2 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T19:13:47,123 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-26T19:13:47,127 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-26T19:13:47,127 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:13:47,128 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T19:13:47,129 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-26T19:13:47,132 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-26T19:13:47,132 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:13:47,134 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T19:13:47,134 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-26T19:13:47,137 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-26T19:13:47,137 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:13:47,138 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T19:13:47,138 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-26T19:13:47,141 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-26T19:13:47,141 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:13:47,142 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T19:13:47,142 DEBUG [PEWorker-2 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-26T19:13:47,143 DEBUG [PEWorker-2 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/hbase/meta/1588230740 2024-11-26T19:13:47,145 DEBUG [PEWorker-2 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/hbase/meta/1588230740 2024-11-26T19:13:47,148 DEBUG [PEWorker-2 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-26T19:13:47,148 DEBUG [PEWorker-2 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-26T19:13:47,149 DEBUG [PEWorker-2 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-26T19:13:47,152 DEBUG [PEWorker-2 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-26T19:13:47,158 DEBUG [PEWorker-2 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T19:13:47,159 INFO [PEWorker-2 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59639694, jitterRate=-0.1112993061542511}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-26T19:13:47,162 DEBUG [PEWorker-2 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732648427120Initializing all the Stores at 1732648427123 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732648427123Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732648427123Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732648427123Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732648427123Cleaning up temporary data from old regions at 1732648427148 (+25 ms)Region opened successfully at 1732648427162 (+14 ms) 2024-11-26T19:13:47,162 DEBUG [PEWorker-2 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-26T19:13:47,163 INFO [PEWorker-2 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-26T19:13:47,163 DEBUG [PEWorker-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-26T19:13:47,163 DEBUG [PEWorker-2 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-26T19:13:47,163 DEBUG [PEWorker-2 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-26T19:13:47,165 INFO [PEWorker-2 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-26T19:13:47,165 DEBUG [PEWorker-2 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732648427162Disabling compacts and flushes for region at 1732648427162Disabling writes for close at 1732648427163 (+1 ms)Writing region close event to WAL at 1732648427164 (+1 ms)Closed at 1732648427165 (+1 ms) 2024-11-26T19:13:47,171 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-26T19:13:47,171 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-26T19:13:47,179 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-26T19:13:47,188 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-26T19:13:47,191 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-26T19:13:47,345 DEBUG [fe76acf732b2:33811 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-26T19:13:47,360 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=fe76acf732b2,42785,1732648423970 2024-11-26T19:13:47,369 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as fe76acf732b2,42785,1732648423970, state=OPENING 2024-11-26T19:13:47,375 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-26T19:13:47,377 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42785-0x10155b2b5d30001, quorum=127.0.0.1:62039, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T19:13:47,377 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33811-0x10155b2b5d30000, quorum=127.0.0.1:62039, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T19:13:47,378 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T19:13:47,379 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T19:13:47,380 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-26T19:13:47,382 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=fe76acf732b2,42785,1732648423970}] 2024-11-26T19:13:47,560 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-26T19:13:47,563 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47227, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-26T19:13:47,573 INFO [RS_OPEN_META-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-26T19:13:47,574 INFO [RS_OPEN_META-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T19:13:47,575 INFO [RS_OPEN_META-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-26T19:13:47,578 INFO [RS_OPEN_META-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=fe76acf732b2%2C42785%2C1732648423970.meta, suffix=.meta, logDir=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/WALs/fe76acf732b2,42785,1732648423970, archiveDir=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/oldWALs, maxLogs=32 2024-11-26T19:13:47,602 DEBUG [RS_OPEN_META-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/WALs/fe76acf732b2,42785,1732648423970/fe76acf732b2%2C42785%2C1732648423970.meta.1732648427581.meta, exclude list is [], retry=0 2024-11-26T19:13:47,605 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34683,DS-754256f3-2cdd-43bf-a093-a98dd3bb7358,DISK] 2024-11-26T19:13:47,608 INFO [RS_OPEN_META-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/WALs/fe76acf732b2,42785,1732648423970/fe76acf732b2%2C42785%2C1732648423970.meta.1732648427581.meta 2024-11-26T19:13:47,609 DEBUG [RS_OPEN_META-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38333:38333)] 2024-11-26T19:13:47,609 DEBUG [RS_OPEN_META-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-26T19:13:47,611 DEBUG [RS_OPEN_META-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-26T19:13:47,614 DEBUG [RS_OPEN_META-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-26T19:13:47,619 INFO [RS_OPEN_META-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-26T19:13:47,623 DEBUG [RS_OPEN_META-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-26T19:13:47,624 DEBUG [RS_OPEN_META-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T19:13:47,624 DEBUG [RS_OPEN_META-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-26T19:13:47,624 DEBUG [RS_OPEN_META-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-26T19:13:47,627 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-26T19:13:47,628 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-26T19:13:47,629 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:13:47,629 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T19:13:47,630 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-26T19:13:47,631 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-26T19:13:47,631 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:13:47,632 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T19:13:47,632 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-26T19:13:47,633 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-26T19:13:47,634 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:13:47,634 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T19:13:47,634 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-26T19:13:47,636 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-26T19:13:47,636 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:13:47,637 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T19:13:47,637 DEBUG [RS_OPEN_META-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-26T19:13:47,638 DEBUG [RS_OPEN_META-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/hbase/meta/1588230740 2024-11-26T19:13:47,640 DEBUG [RS_OPEN_META-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/hbase/meta/1588230740 2024-11-26T19:13:47,643 DEBUG [RS_OPEN_META-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-26T19:13:47,643 DEBUG [RS_OPEN_META-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-26T19:13:47,644 DEBUG [RS_OPEN_META-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-26T19:13:47,647 DEBUG [RS_OPEN_META-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-26T19:13:47,648 INFO [RS_OPEN_META-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64117199, jitterRate=-0.04457928240299225}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-26T19:13:47,649 DEBUG [RS_OPEN_META-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-26T19:13:47,651 DEBUG [RS_OPEN_META-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732648427625Writing region info on filesystem at 1732648427625Initializing all the Stores at 1732648427627 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732648427627Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732648427627Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732648427627Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732648427627Cleaning up temporary data from old regions at 1732648427643 (+16 ms)Running coprocessor post-open hooks at 1732648427649 (+6 ms)Region opened successfully at 1732648427651 (+2 ms) 2024-11-26T19:13:47,659 INFO [RS_OPEN_META-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732648427549 2024-11-26T19:13:47,672 DEBUG [RS_OPEN_META-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-26T19:13:47,672 INFO [RS_OPEN_META-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-26T19:13:47,674 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=fe76acf732b2,42785,1732648423970 2024-11-26T19:13:47,676 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as fe76acf732b2,42785,1732648423970, state=OPEN 2024-11-26T19:13:47,682 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33811-0x10155b2b5d30000, quorum=127.0.0.1:62039, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T19:13:47,682 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42785-0x10155b2b5d30001, quorum=127.0.0.1:62039, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T19:13:47,682 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T19:13:47,682 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T19:13:47,682 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:13:47,688 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-26T19:13:47,688 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=fe76acf732b2,42785,1732648423970 in 301 msec 2024-11-26T19:13:47,693 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-26T19:13:47,693 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 511 msec 2024-11-26T19:13:47,695 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-26T19:13:47,695 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-26T19:13:47,716 DEBUG [PEWorker-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T19:13:47,717 DEBUG [PEWorker-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fe76acf732b2,42785,1732648423970, seqNum=-1] 2024-11-26T19:13:47,738 DEBUG [PEWorker-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T19:13:47,741 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45485, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T19:13:47,763 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.5910 sec 2024-11-26T19:13:47,763 INFO [master/fe76acf732b2:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732648427763, completionTime=-1 2024-11-26T19:13:47,766 INFO [master/fe76acf732b2:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-26T19:13:47,766 DEBUG [master/fe76acf732b2:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-26T19:13:47,791 INFO [master/fe76acf732b2:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-26T19:13:47,791 INFO [master/fe76acf732b2:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732648487791 2024-11-26T19:13:47,792 INFO [master/fe76acf732b2:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732648547792 2024-11-26T19:13:47,792 INFO [master/fe76acf732b2:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 25 msec 2024-11-26T19:13:47,794 INFO [master/fe76acf732b2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fe76acf732b2,33811,1732648423099-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T19:13:47,794 INFO [master/fe76acf732b2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fe76acf732b2,33811,1732648423099-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T19:13:47,794 INFO [master/fe76acf732b2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fe76acf732b2,33811,1732648423099-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T19:13:47,796 INFO [master/fe76acf732b2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-fe76acf732b2:33811, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T19:13:47,796 INFO [master/fe76acf732b2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-26T19:13:47,797 INFO [master/fe76acf732b2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-26T19:13:47,802 DEBUG [master/fe76acf732b2:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-26T19:13:47,826 INFO [master/fe76acf732b2:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 3.661sec 2024-11-26T19:13:47,827 INFO [master/fe76acf732b2:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-26T19:13:47,829 INFO [master/fe76acf732b2:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-26T19:13:47,830 INFO [master/fe76acf732b2:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-26T19:13:47,830 INFO [master/fe76acf732b2:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-26T19:13:47,831 INFO [master/fe76acf732b2:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-26T19:13:47,831 INFO [master/fe76acf732b2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fe76acf732b2,33811,1732648423099-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-26T19:13:47,832 INFO [master/fe76acf732b2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fe76acf732b2,33811,1732648423099-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-26T19:13:47,841 DEBUG [master/fe76acf732b2:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-26T19:13:47,842 INFO [master/fe76acf732b2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-26T19:13:47,843 INFO [master/fe76acf732b2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fe76acf732b2,33811,1732648423099-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T19:13:47,921 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d57a2b9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:13:47,924 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-26T19:13:47,924 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-26T19:13:47,928 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fe76acf732b2,33811,-1 for getting cluster id 2024-11-26T19:13:47,931 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T19:13:47,940 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3cec4e24-71db-4cd0-b63a-4ea30a9b160b' 2024-11-26T19:13:47,943 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T19:13:47,944 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3cec4e24-71db-4cd0-b63a-4ea30a9b160b" 2024-11-26T19:13:47,946 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a872003, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:13:47,947 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fe76acf732b2,33811,-1] 2024-11-26T19:13:47,950 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T19:13:47,952 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:13:47,954 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42804, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T19:13:47,957 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a7fbb0f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:13:47,958 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T19:13:47,966 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fe76acf732b2,42785,1732648423970, seqNum=-1] 2024-11-26T19:13:47,967 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T19:13:47,969 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49098, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T19:13:47,997 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=fe76acf732b2,33811,1732648423099 2024-11-26T19:13:48,013 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=357, ProcessCount=11, AvailableMemoryMB=6774 2024-11-26T19:13:48,025 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-26T19:13:48,033 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is fe76acf732b2,33811,1732648423099 2024-11-26T19:13:48,058 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@54765f80 2024-11-26T19:13:48,060 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-26T19:13:48,068 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42818, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-26T19:13:48,071 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33811 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-26T19:13:48,077 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33811 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-26T19:13:48,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33811 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-26T19:13:48,102 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-26T19:13:48,104 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33811 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-26T19:13:48,105 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:13:48,108 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-26T19:13:48,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-26T19:13:48,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741835_1011 (size=1061) 2024-11-26T19:13:48,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-26T19:13:48,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-26T19:13:48,532 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16 2024-11-26T19:13:48,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741836_1012 (size=53) 2024-11-26T19:13:48,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-26T19:13:48,942 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T19:13:48,942 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 76c4ff0c848e5ffb5ad9989537063596, disabling compactions & flushes 2024-11-26T19:13:48,942 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. 2024-11-26T19:13:48,942 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. 2024-11-26T19:13:48,942 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. after waiting 0 ms 2024-11-26T19:13:48,942 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. 2024-11-26T19:13:48,942 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. 2024-11-26T19:13:48,942 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 76c4ff0c848e5ffb5ad9989537063596: Waiting for close lock at 1732648428942Disabling compacts and flushes for region at 1732648428942Disabling writes for close at 1732648428942Writing region close event to WAL at 1732648428942Closed at 1732648428942 2024-11-26T19:13:48,945 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-26T19:13:48,952 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732648428946"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732648428946"}]},"ts":"1732648428946"} 2024-11-26T19:13:48,959 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-26T19:13:48,961 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-26T19:13:48,964 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732648428961"}]},"ts":"1732648428961"} 2024-11-26T19:13:48,970 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-26T19:13:48,973 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=76c4ff0c848e5ffb5ad9989537063596, ASSIGN}] 2024-11-26T19:13:48,976 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=76c4ff0c848e5ffb5ad9989537063596, ASSIGN 2024-11-26T19:13:48,978 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=76c4ff0c848e5ffb5ad9989537063596, ASSIGN; state=OFFLINE, location=fe76acf732b2,42785,1732648423970; forceNewPlan=false, retain=false 2024-11-26T19:13:49,130 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=76c4ff0c848e5ffb5ad9989537063596, regionState=OPENING, regionLocation=fe76acf732b2,42785,1732648423970 2024-11-26T19:13:49,135 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=76c4ff0c848e5ffb5ad9989537063596, ASSIGN because future has completed 2024-11-26T19:13:49,137 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970}] 2024-11-26T19:13:49,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-26T19:13:49,303 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. 2024-11-26T19:13:49,303 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} 2024-11-26T19:13:49,304 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 76c4ff0c848e5ffb5ad9989537063596 2024-11-26T19:13:49,304 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T19:13:49,304 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 76c4ff0c848e5ffb5ad9989537063596 2024-11-26T19:13:49,304 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 76c4ff0c848e5ffb5ad9989537063596 2024-11-26T19:13:49,307 INFO [StoreOpener-76c4ff0c848e5ffb5ad9989537063596-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 76c4ff0c848e5ffb5ad9989537063596 2024-11-26T19:13:49,311 INFO [StoreOpener-76c4ff0c848e5ffb5ad9989537063596-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-26T19:13:49,311 INFO [StoreOpener-76c4ff0c848e5ffb5ad9989537063596-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 76c4ff0c848e5ffb5ad9989537063596 columnFamilyName A 2024-11-26T19:13:49,311 DEBUG [StoreOpener-76c4ff0c848e5ffb5ad9989537063596-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:13:49,313 INFO [StoreOpener-76c4ff0c848e5ffb5ad9989537063596-1 {}] regionserver.HStore(327): Store=76c4ff0c848e5ffb5ad9989537063596/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T19:13:49,313 INFO [StoreOpener-76c4ff0c848e5ffb5ad9989537063596-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 76c4ff0c848e5ffb5ad9989537063596 2024-11-26T19:13:49,315 INFO [StoreOpener-76c4ff0c848e5ffb5ad9989537063596-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-26T19:13:49,316 INFO [StoreOpener-76c4ff0c848e5ffb5ad9989537063596-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 76c4ff0c848e5ffb5ad9989537063596 columnFamilyName B 2024-11-26T19:13:49,316 DEBUG [StoreOpener-76c4ff0c848e5ffb5ad9989537063596-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:13:49,317 INFO [StoreOpener-76c4ff0c848e5ffb5ad9989537063596-1 {}] regionserver.HStore(327): Store=76c4ff0c848e5ffb5ad9989537063596/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T19:13:49,317 INFO [StoreOpener-76c4ff0c848e5ffb5ad9989537063596-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 76c4ff0c848e5ffb5ad9989537063596 2024-11-26T19:13:49,319 INFO [StoreOpener-76c4ff0c848e5ffb5ad9989537063596-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-26T19:13:49,319 INFO [StoreOpener-76c4ff0c848e5ffb5ad9989537063596-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 76c4ff0c848e5ffb5ad9989537063596 columnFamilyName C 2024-11-26T19:13:49,319 DEBUG [StoreOpener-76c4ff0c848e5ffb5ad9989537063596-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:13:49,320 INFO [StoreOpener-76c4ff0c848e5ffb5ad9989537063596-1 {}] regionserver.HStore(327): Store=76c4ff0c848e5ffb5ad9989537063596/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T19:13:49,321 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. 2024-11-26T19:13:49,321 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 76c4ff0c848e5ffb5ad9989537063596 2024-11-26T19:13:49,323 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596 2024-11-26T19:13:49,323 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596 2024-11-26T19:13:49,326 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 76c4ff0c848e5ffb5ad9989537063596 2024-11-26T19:13:49,326 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 76c4ff0c848e5ffb5ad9989537063596 2024-11-26T19:13:49,328 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-26T19:13:49,331 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 76c4ff0c848e5ffb5ad9989537063596 2024-11-26T19:13:49,346 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T19:13:49,347 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 76c4ff0c848e5ffb5ad9989537063596; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62576451, jitterRate=-0.06753821671009064}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-26T19:13:49,347 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 76c4ff0c848e5ffb5ad9989537063596 2024-11-26T19:13:49,349 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 76c4ff0c848e5ffb5ad9989537063596: Running coprocessor pre-open hook at 1732648429304Writing region info on filesystem at 1732648429304Initializing all the Stores at 1732648429306 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732648429306Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732648429306Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732648429306Cleaning up temporary data from old regions at 1732648429326 (+20 ms)Running coprocessor post-open hooks at 1732648429347 (+21 ms)Region opened successfully at 1732648429348 (+1 ms) 2024-11-26T19:13:49,351 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., pid=6, masterSystemTime=1732648429292 2024-11-26T19:13:49,356 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. 2024-11-26T19:13:49,356 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. 2024-11-26T19:13:49,358 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=76c4ff0c848e5ffb5ad9989537063596, regionState=OPEN, openSeqNum=2, regionLocation=fe76acf732b2,42785,1732648423970 2024-11-26T19:13:49,362 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 because future has completed 2024-11-26T19:13:49,370 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-26T19:13:49,370 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 in 227 msec 2024-11-26T19:13:49,376 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-26T19:13:49,377 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=76c4ff0c848e5ffb5ad9989537063596, ASSIGN in 397 msec 2024-11-26T19:13:49,378 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-26T19:13:49,379 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732648429378"}]},"ts":"1732648429378"} 2024-11-26T19:13:49,382 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-26T19:13:49,384 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-26T19:13:49,388 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.3040 sec 2024-11-26T19:13:50,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-26T19:13:50,274 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-26T19:13:50,280 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1487a66f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:13:50,280 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fe76acf732b2,33811,-1 for getting cluster id 2024-11-26T19:13:50,280 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T19:13:50,282 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3cec4e24-71db-4cd0-b63a-4ea30a9b160b' 2024-11-26T19:13:50,282 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T19:13:50,282 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3cec4e24-71db-4cd0-b63a-4ea30a9b160b" 2024-11-26T19:13:50,283 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12f6a966, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:13:50,283 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fe76acf732b2,33811,-1] 2024-11-26T19:13:50,283 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T19:13:50,283 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:13:50,285 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42834, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T19:13:50,286 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5cc7db18, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:13:50,287 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T19:13:50,288 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fe76acf732b2,42785,1732648423970, seqNum=-1] 2024-11-26T19:13:50,288 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T19:13:50,293 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49100, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T19:13:50,296 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-26T19:13:50,297 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is fe76acf732b2,33811,1732648423099 2024-11-26T19:13:50,298 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@5f70fa58 2024-11-26T19:13:50,300 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-26T19:13:50,302 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42838, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-26T19:13:50,308 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50d3f2f4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:13:50,308 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fe76acf732b2,33811,-1 for getting cluster id 2024-11-26T19:13:50,309 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T19:13:50,310 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3cec4e24-71db-4cd0-b63a-4ea30a9b160b' 2024-11-26T19:13:50,311 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T19:13:50,311 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3cec4e24-71db-4cd0-b63a-4ea30a9b160b" 2024-11-26T19:13:50,311 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e6ed1ec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:13:50,311 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fe76acf732b2,33811,-1] 2024-11-26T19:13:50,312 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T19:13:50,312 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:13:50,314 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42854, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T19:13:50,315 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33bdb6a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:13:50,316 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46ad9e2c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:13:50,316 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fe76acf732b2,33811,-1 for getting cluster id 2024-11-26T19:13:50,316 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T19:13:50,320 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3cec4e24-71db-4cd0-b63a-4ea30a9b160b' 2024-11-26T19:13:50,321 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T19:13:50,321 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3cec4e24-71db-4cd0-b63a-4ea30a9b160b" 2024-11-26T19:13:50,321 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fa64271, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:13:50,321 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fe76acf732b2,33811,-1] 2024-11-26T19:13:50,322 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T19:13:50,324 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:13:50,325 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42866, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T19:13:50,341 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27108bea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:13:50,343 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6308d33d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:13:50,343 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fe76acf732b2,33811,-1 for getting cluster id 2024-11-26T19:13:50,343 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T19:13:50,347 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3cec4e24-71db-4cd0-b63a-4ea30a9b160b' 2024-11-26T19:13:50,348 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T19:13:50,348 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3cec4e24-71db-4cd0-b63a-4ea30a9b160b" 2024-11-26T19:13:50,349 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ec6ad86, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:13:50,349 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fe76acf732b2,33811,-1] 2024-11-26T19:13:50,349 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T19:13:50,349 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:13:50,351 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42896, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T19:13:50,352 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32b0e30d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:13:50,353 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a822d7a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:13:50,353 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fe76acf732b2,33811,-1 for getting cluster id 2024-11-26T19:13:50,354 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T19:13:50,355 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3cec4e24-71db-4cd0-b63a-4ea30a9b160b' 2024-11-26T19:13:50,355 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T19:13:50,355 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3cec4e24-71db-4cd0-b63a-4ea30a9b160b" 2024-11-26T19:13:50,356 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4545009e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:13:50,356 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fe76acf732b2,33811,-1] 2024-11-26T19:13:50,356 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T19:13:50,357 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:13:50,358 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42910, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T19:13:50,360 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31097510, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:13:50,364 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7813eeba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:13:50,364 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fe76acf732b2,33811,-1 for getting cluster id 2024-11-26T19:13:50,365 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T19:13:50,366 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3cec4e24-71db-4cd0-b63a-4ea30a9b160b' 2024-11-26T19:13:50,366 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T19:13:50,366 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3cec4e24-71db-4cd0-b63a-4ea30a9b160b" 2024-11-26T19:13:50,367 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f695ce, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:13:50,367 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fe76acf732b2,33811,-1] 2024-11-26T19:13:50,367 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T19:13:50,367 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:13:50,368 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42922, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T19:13:50,369 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a60eb72, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:13:50,372 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f66d26b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:13:50,372 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fe76acf732b2,33811,-1 for getting cluster id 2024-11-26T19:13:50,373 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T19:13:50,373 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3cec4e24-71db-4cd0-b63a-4ea30a9b160b' 2024-11-26T19:13:50,374 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T19:13:50,374 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3cec4e24-71db-4cd0-b63a-4ea30a9b160b" 2024-11-26T19:13:50,375 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b3c1f14, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:13:50,375 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fe76acf732b2,33811,-1] 2024-11-26T19:13:50,375 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T19:13:50,375 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:13:50,376 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42928, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T19:13:50,377 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32e1b379, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:13:50,384 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@225d1960, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:13:50,385 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fe76acf732b2,33811,-1 for getting cluster id 2024-11-26T19:13:50,385 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T19:13:50,386 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3cec4e24-71db-4cd0-b63a-4ea30a9b160b' 2024-11-26T19:13:50,386 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T19:13:50,386 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3cec4e24-71db-4cd0-b63a-4ea30a9b160b" 2024-11-26T19:13:50,387 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@89bfc1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:13:50,387 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fe76acf732b2,33811,-1] 2024-11-26T19:13:50,387 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T19:13:50,388 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:13:50,389 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42944, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T19:13:50,390 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46d3547a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:13:50,391 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@93b4e10, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:13:50,391 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fe76acf732b2,33811,-1 for getting cluster id 2024-11-26T19:13:50,391 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T19:13:50,392 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3cec4e24-71db-4cd0-b63a-4ea30a9b160b' 2024-11-26T19:13:50,392 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T19:13:50,392 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3cec4e24-71db-4cd0-b63a-4ea30a9b160b" 2024-11-26T19:13:50,393 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6cb5cb2d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:13:50,393 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fe76acf732b2,33811,-1] 2024-11-26T19:13:50,393 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T19:13:50,394 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:13:50,395 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42966, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T19:13:50,396 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3433011f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:13:50,400 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7449cb3c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:13:50,400 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fe76acf732b2,33811,-1 for getting cluster id 2024-11-26T19:13:50,400 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T19:13:50,401 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3cec4e24-71db-4cd0-b63a-4ea30a9b160b' 2024-11-26T19:13:50,402 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T19:13:50,402 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3cec4e24-71db-4cd0-b63a-4ea30a9b160b" 2024-11-26T19:13:50,402 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58b0f1b2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:13:50,402 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fe76acf732b2,33811,-1] 2024-11-26T19:13:50,403 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T19:13:50,403 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:13:50,404 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42984, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T19:13:50,405 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66945757, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:13:50,417 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33811 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T19:13:50,418 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T19:13:50,421 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fe76acf732b2,42785,1732648423970, seqNum=-1] 2024-11-26T19:13:50,422 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T19:13:50,423 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T19:13:50,424 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49116, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T19:13:50,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33811 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-26T19:13:50,426 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fe76acf732b2,42785,1732648423970, seqNum=-1] 2024-11-26T19:13:50,426 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T19:13:50,429 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T19:13:50,429 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49118, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T19:13:50,430 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T19:13:50,431 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T19:13:50,432 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fe76acf732b2,42785,1732648423970, seqNum=-1] 2024-11-26T19:13:50,432 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T19:13:50,433 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T19:13:50,434 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2] 2024-11-26T19:13:50,434 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49132, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T19:13:50,434 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2] 2024-11-26T19:13:50,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-26T19:13:50,437 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T19:13:50,438 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2] 2024-11-26T19:13:50,439 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fe76acf732b2,42785,1732648423970, seqNum=-1] 2024-11-26T19:13:50,439 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T19:13:50,441 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49146, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T19:13:50,445 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2] 2024-11-26T19:13:50,450 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T19:13:50,451 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fe76acf732b2,42785,1732648423970, seqNum=-1] 2024-11-26T19:13:50,452 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T19:13:50,453 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T19:13:50,454 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49154, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T19:13:50,455 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T19:13:50,457 DEBUG [Thread-154 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fe76acf732b2,42785,1732648423970, seqNum=-1] 2024-11-26T19:13:50,457 DEBUG [Thread-154 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T19:13:50,457 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2] 2024-11-26T19:13:50,461 DEBUG [Thread-145 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fe76acf732b2,42785,1732648423970, seqNum=-1] 2024-11-26T19:13:50,461 DEBUG [Thread-145 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T19:13:50,462 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49168, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T19:13:50,465 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49182, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T19:13:50,465 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2] 2024-11-26T19:13:50,467 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T19:13:50,468 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fe76acf732b2,42785,1732648423970, seqNum=-1] 2024-11-26T19:13:50,468 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2] 2024-11-26T19:13:50,469 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T19:13:50,471 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49198, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T19:13:50,474 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T19:13:50,474 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2] 2024-11-26T19:13:50,476 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fe76acf732b2,42785,1732648423970, seqNum=-1] 2024-11-26T19:13:50,476 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T19:13:50,486 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49210, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T19:13:50,490 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2] 2024-11-26T19:13:50,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-26T19:13:50,595 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 76c4ff0c848e5ffb5ad9989537063596 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-26T19:13:50,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(8855): Flush requested on 76c4ff0c848e5ffb5ad9989537063596 2024-11-26T19:13:50,604 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c4ff0c848e5ffb5ad9989537063596, store=A 2024-11-26T19:13:50,604 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:13:50,605 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c4ff0c848e5ffb5ad9989537063596, store=B 2024-11-26T19:13:50,605 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:13:50,605 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c4ff0c848e5ffb5ad9989537063596, store=C 2024-11-26T19:13:50,605 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:13:50,619 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-26T19:13:50,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. 2024-11-26T19:13:50,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. as already flushing 2024-11-26T19:13:50,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. 2024-11-26T19:13:50,639 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:13:50,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:13:50,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:13:50,728 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/A/a198223cb57a4570b8cb1128e123f979 is 50, key is test_row_0/A:col10/1732648430555/Put/seqid=0 2024-11-26T19:13:50,746 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732648430686, max timestamp=1732648430724Num uniques 859; 2024-11-26T19:13:50,747 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-26T19:13:50,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-26T19:13:50,758 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732648430686, max timestamp=1732648430724Num uniques 859; 2024-11-26T19:13:50,758 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-26T19:13:50,762 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732648430686, max timestamp=1732648430740Num uniques 859; 2024-11-26T19:13:50,762 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-26T19:13:50,784 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741837_1013 (size=12001) 2024-11-26T19:13:50,816 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:13:50,820 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:13:50,822 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:13:50,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49132 deadline: 1732648490800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:13:50,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49182 deadline: 1732648490800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:13:50,822 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-26T19:13:50,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49210 deadline: 1732648490804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:13:50,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. 2024-11-26T19:13:50,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. as already flushing 2024-11-26T19:13:50,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. 2024-11-26T19:13:50,826 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:13:50,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:13:50,827 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:13:50,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49154 deadline: 1732648490804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:13:50,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:13:50,834 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:13:50,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49198 deadline: 1732648490821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:13:50,857 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:50,857 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:50,857 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:50,857 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:50,858 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:50,858 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:50,858 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:50,859 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:50,859 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:13:50,858 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:13:50,858 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:13:50,859 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:13:50,860 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:50,860 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:50,861 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:13:50,964 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:13:50,965 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:13:50,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49198 deadline: 1732648490963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:13:50,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49182 deadline: 1732648490963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:13:50,966 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:50,966 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:50,966 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:50,967 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:13:50,966 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:50,967 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:13:50,968 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:13:50,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49132 deadline: 1732648490964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:13:50,969 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:50,969 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:50,969 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:13:50,970 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:13:50,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49154 deadline: 1732648490965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:13:50,971 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:50,971 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:50,971 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:13:50,977 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:13:50,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49210 deadline: 1732648490975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:13:50,979 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:50,979 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:50,979 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:13:50,984 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-26T19:13:50,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. 2024-11-26T19:13:50,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. as already flushing 2024-11-26T19:13:50,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. 2024-11-26T19:13:50,989 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:13:50,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:13:50,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:13:51,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-26T19:13:51,146 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-26T19:13:51,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. 2024-11-26T19:13:51,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. as already flushing 2024-11-26T19:13:51,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. 2024-11-26T19:13:51,147 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:13:51,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:13:51,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:13:51,174 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:13:51,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49198 deadline: 1732648491173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:13:51,176 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:13:51,176 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:51,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49182 deadline: 1732648491173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:13:51,177 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:51,177 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:13:51,177 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:51,178 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:51,178 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:13:51,189 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:13:51,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49132 deadline: 1732648491183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:13:51,189 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/A/a198223cb57a4570b8cb1128e123f979 2024-11-26T19:13:51,189 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:13:51,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49154 deadline: 1732648491183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:13:51,190 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:51,190 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:51,190 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:13:51,190 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:51,190 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:51,190 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:13:51,196 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:13:51,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49210 deadline: 1732648491193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:13:51,197 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:51,197 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:51,197 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:13:51,303 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-26T19:13:51,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. 2024-11-26T19:13:51,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. as already flushing 2024-11-26T19:13:51,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. 2024-11-26T19:13:51,304 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:13:51,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:13:51,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:13:51,312 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/B/caee9bb5427744818d43a7ad8a612201 is 50, key is test_row_0/B:col10/1732648430555/Put/seqid=0 2024-11-26T19:13:51,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741838_1014 (size=12001) 2024-11-26T19:13:51,458 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-26T19:13:51,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. 2024-11-26T19:13:51,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. as already flushing 2024-11-26T19:13:51,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. 2024-11-26T19:13:51,459 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:13:51,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:13:51,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:13:51,486 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:13:51,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49198 deadline: 1732648491483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:13:51,487 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:51,487 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:51,488 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:13:51,489 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:13:51,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49182 deadline: 1732648491483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:13:51,490 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:51,490 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:51,490 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:13:51,504 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:13:51,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49154 deadline: 1732648491504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:13:51,506 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:51,506 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:51,506 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:13:51,507 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:13:51,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49132 deadline: 1732648491506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:13:51,509 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:51,509 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:51,509 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:13:51,509 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:13:51,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49210 deadline: 1732648491506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:13:51,510 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:51,510 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:51,510 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:13:51,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-26T19:13:51,614 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-26T19:13:51,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. 2024-11-26T19:13:51,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. as already flushing 2024-11-26T19:13:51,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. 2024-11-26T19:13:51,615 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:13:51,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:13:51,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:13:51,750 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/B/caee9bb5427744818d43a7ad8a612201 2024-11-26T19:13:51,774 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-26T19:13:51,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. 2024-11-26T19:13:51,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. as already flushing 2024-11-26T19:13:51,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. 2024-11-26T19:13:51,783 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:13:51,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:13:51,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:13:51,794 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/C/4b5236a5c5d14cd6bf4d3b06b23d2f94 is 50, key is test_row_0/C:col10/1732648430555/Put/seqid=0 2024-11-26T19:13:51,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741839_1015 (size=12001) 2024-11-26T19:13:51,939 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-26T19:13:51,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. 2024-11-26T19:13:51,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. as already flushing 2024-11-26T19:13:51,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. 2024-11-26T19:13:51,941 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:13:51,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:13:51,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:13:52,007 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:13:52,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49198 deadline: 1732648492002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:13:52,007 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:13:52,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49182 deadline: 1732648492002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:13:52,008 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:52,008 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:52,008 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:13:52,008 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:52,009 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:52,009 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:13:52,014 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:13:52,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49154 deadline: 1732648492012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:13:52,015 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:52,015 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:52,015 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:13:52,025 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:13:52,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49132 deadline: 1732648492023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:13:52,027 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:13:52,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49210 deadline: 1732648492023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:13:52,028 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:52,028 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:52,028 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:13:52,029 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:52,029 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:52,029 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:13:52,101 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-26T19:13:52,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. 2024-11-26T19:13:52,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. as already flushing 2024-11-26T19:13:52,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. 2024-11-26T19:13:52,102 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:13:52,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:13:52,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:13:52,208 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/C/4b5236a5c5d14cd6bf4d3b06b23d2f94 2024-11-26T19:13:52,225 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/A/a198223cb57a4570b8cb1128e123f979 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/a198223cb57a4570b8cb1128e123f979 2024-11-26T19:13:52,239 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/a198223cb57a4570b8cb1128e123f979, entries=150, sequenceid=12, filesize=11.7 K 2024-11-26T19:13:52,245 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/B/caee9bb5427744818d43a7ad8a612201 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/caee9bb5427744818d43a7ad8a612201 2024-11-26T19:13:52,257 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-26T19:13:52,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. 2024-11-26T19:13:52,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. as already flushing 2024-11-26T19:13:52,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. 2024-11-26T19:13:52,258 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:13:52,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:13:52,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:13:52,266 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/caee9bb5427744818d43a7ad8a612201, entries=150, sequenceid=12, filesize=11.7 K 2024-11-26T19:13:52,269 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/C/4b5236a5c5d14cd6bf4d3b06b23d2f94 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/4b5236a5c5d14cd6bf4d3b06b23d2f94 2024-11-26T19:13:52,281 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/4b5236a5c5d14cd6bf4d3b06b23d2f94, entries=150, sequenceid=12, filesize=11.7 K 2024-11-26T19:13:52,287 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 76c4ff0c848e5ffb5ad9989537063596 in 1692ms, sequenceid=12, compaction requested=false 2024-11-26T19:13:52,289 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-26T19:13:52,292 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 76c4ff0c848e5ffb5ad9989537063596: 2024-11-26T19:13:52,373 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-26T19:13:52,413 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-26T19:13:52,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. 2024-11-26T19:13:52,414 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing 76c4ff0c848e5ffb5ad9989537063596 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-11-26T19:13:52,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c4ff0c848e5ffb5ad9989537063596, store=A 2024-11-26T19:13:52,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T19:13:52,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c4ff0c848e5ffb5ad9989537063596, store=B 2024-11-26T19:13:52,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T19:13:52,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c4ff0c848e5ffb5ad9989537063596, store=C 2024-11-26T19:13:52,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T19:13:52,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/A/eab18453588b45ddad179336a0d7ccd4 is 50, key is test_row_0/A:col10/1732648430795/Put/seqid=0 2024-11-26T19:13:52,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741840_1016 (size=12001) 2024-11-26T19:13:52,456 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/A/eab18453588b45ddad179336a0d7ccd4 2024-11-26T19:13:52,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/B/ad268253061e43b88ec365621129766a is 50, key is test_row_0/B:col10/1732648430795/Put/seqid=0 2024-11-26T19:13:52,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741841_1017 (size=12001) 2024-11-26T19:13:52,525 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/B/ad268253061e43b88ec365621129766a 2024-11-26T19:13:52,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/C/1beea36d53264c0cb8b3f76dcac2c531 is 50, key is test_row_0/C:col10/1732648430795/Put/seqid=0 2024-11-26T19:13:52,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-26T19:13:52,603 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741842_1018 (size=12001) 2024-11-26T19:13:52,606 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/C/1beea36d53264c0cb8b3f76dcac2c531 2024-11-26T19:13:52,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/A/eab18453588b45ddad179336a0d7ccd4 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/eab18453588b45ddad179336a0d7ccd4 2024-11-26T19:13:52,639 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/eab18453588b45ddad179336a0d7ccd4, entries=150, sequenceid=47, filesize=11.7 K 2024-11-26T19:13:52,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/B/ad268253061e43b88ec365621129766a as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/ad268253061e43b88ec365621129766a 2024-11-26T19:13:52,657 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/ad268253061e43b88ec365621129766a, entries=150, sequenceid=47, filesize=11.7 K 2024-11-26T19:13:52,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/C/1beea36d53264c0cb8b3f76dcac2c531 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/1beea36d53264c0cb8b3f76dcac2c531 2024-11-26T19:13:52,672 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/1beea36d53264c0cb8b3f76dcac2c531, entries=150, sequenceid=47, filesize=11.7 K 2024-11-26T19:13:52,676 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=0 B/0 for 76c4ff0c848e5ffb5ad9989537063596 in 261ms, sequenceid=47, compaction requested=false 2024-11-26T19:13:52,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for 76c4ff0c848e5ffb5ad9989537063596: 2024-11-26T19:13:52,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. 2024-11-26T19:13:52,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-26T19:13:52,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33811 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-26T19:13:52,689 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-26T19:13:52,689 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2460 sec 2024-11-26T19:13:52,693 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 2.2720 sec 2024-11-26T19:13:53,041 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 76c4ff0c848e5ffb5ad9989537063596 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-26T19:13:53,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(8855): Flush requested on 76c4ff0c848e5ffb5ad9989537063596 2024-11-26T19:13:53,049 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c4ff0c848e5ffb5ad9989537063596, store=A 2024-11-26T19:13:53,049 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:13:53,050 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c4ff0c848e5ffb5ad9989537063596, store=B 2024-11-26T19:13:53,050 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:13:53,050 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c4ff0c848e5ffb5ad9989537063596, store=C 2024-11-26T19:13:53,050 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:13:53,062 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/A/1143addfdb464bfbaf1996658cd75569 is 50, key is test_row_0/A:col10/1732648433035/Put/seqid=0 2024-11-26T19:13:53,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741843_1019 (size=14341) 2024-11-26T19:13:53,100 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=58 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/A/1143addfdb464bfbaf1996658cd75569 2024-11-26T19:13:53,121 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:13:53,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49182 deadline: 1732648493103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:13:53,121 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:13:53,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49132 deadline: 1732648493103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:13:53,122 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:53,122 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:53,123 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:13:53,123 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:53,123 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:53,123 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:13:53,123 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732648433073, max timestamp=1732648433113Num uniques 859; 2024-11-26T19:13:53,124 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-26T19:13:53,126 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732648433073, max timestamp=1732648433113Num uniques 859; 2024-11-26T19:13:53,127 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-26T19:13:53,127 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/B/84313ca47744468780bcf3b82b5296af is 50, key is test_row_0/B:col10/1732648433035/Put/seqid=0 2024-11-26T19:13:53,135 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-26T19:13:53,139 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1732648433095, max timestamp=1732648433113Num uniques 859; 2024-11-26T19:13:53,139 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-26T19:13:53,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741844_1020 (size=12001) 2024-11-26T19:13:53,174 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=58 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/B/84313ca47744468780bcf3b82b5296af 2024-11-26T19:13:53,198 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/C/c72b1f40d9a84d0a8c0ce7fbd4efb161 is 50, key is test_row_0/C:col10/1732648433035/Put/seqid=0 2024-11-26T19:13:53,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741845_1021 (size=12001) 2024-11-26T19:13:53,262 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:13:53,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49132 deadline: 1732648493233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:13:53,263 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:53,263 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:13:53,263 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:53,263 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:13:53,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49182 deadline: 1732648493235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:13:53,265 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:53,265 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:53,265 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:13:53,265 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:13:53,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49198 deadline: 1732648493255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:13:53,266 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:53,266 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:13:53,266 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:53,266 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:13:53,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49210 deadline: 1732648493255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:13:53,267 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:13:53,267 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:53,267 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:53,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49154 deadline: 1732648493255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:13:53,267 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:13:53,268 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:53,268 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:53,268 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:13:53,376 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:13:53,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49198 deadline: 1732648493373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:13:53,377 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:53,377 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:13:53,377 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:53,377 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:13:53,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49210 deadline: 1732648493373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:13:53,378 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:53,378 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:13:53,378 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:53,378 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:13:53,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49154 deadline: 1732648493373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:13:53,378 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:53,379 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:53,379 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:13:53,474 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:13:53,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49132 deadline: 1732648493472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:13:53,475 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:13:53,476 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:53,476 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:53,476 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:13:53,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49182 deadline: 1732648493473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:13:53,478 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:53,478 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:53,478 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:13:53,532 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-26T19:13:53,533 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-26T19:13:53,535 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-26T19:13:53,535 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-26T19:13:53,535 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-26T19:13:53,535 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-26T19:13:53,537 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-26T19:13:53,537 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-26T19:13:53,587 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:13:53,587 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:13:53,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49210 deadline: 1732648493583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:13:53,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49198 deadline: 1732648493583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:13:53,588 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:13:53,589 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:53,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49154 deadline: 1732648493584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:13:53,589 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:53,589 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:13:53,589 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:53,589 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:53,589 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:53,590 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:13:53,590 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:53,590 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:13:53,611 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=58 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/C/c72b1f40d9a84d0a8c0ce7fbd4efb161 2024-11-26T19:13:53,626 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/A/1143addfdb464bfbaf1996658cd75569 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/1143addfdb464bfbaf1996658cd75569 2024-11-26T19:13:53,642 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/1143addfdb464bfbaf1996658cd75569, entries=200, sequenceid=58, filesize=14.0 K 2024-11-26T19:13:53,644 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/B/84313ca47744468780bcf3b82b5296af as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/84313ca47744468780bcf3b82b5296af 2024-11-26T19:13:53,656 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/84313ca47744468780bcf3b82b5296af, entries=150, sequenceid=58, filesize=11.7 K 2024-11-26T19:13:53,658 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/C/c72b1f40d9a84d0a8c0ce7fbd4efb161 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/c72b1f40d9a84d0a8c0ce7fbd4efb161 2024-11-26T19:13:53,668 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/c72b1f40d9a84d0a8c0ce7fbd4efb161, entries=150, sequenceid=58, filesize=11.7 K 2024-11-26T19:13:53,670 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=133.78 KB/136995 for 76c4ff0c848e5ffb5ad9989537063596 in 630ms, sequenceid=58, compaction requested=true 2024-11-26T19:13:53,671 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 76c4ff0c848e5ffb5ad9989537063596: 2024-11-26T19:13:53,676 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 76c4ff0c848e5ffb5ad9989537063596:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T19:13:53,679 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T19:13:53,680 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:13:53,681 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T19:13:53,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 76c4ff0c848e5ffb5ad9989537063596:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T19:13:53,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:13:53,685 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 76c4ff0c848e5ffb5ad9989537063596:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T19:13:53,685 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T19:13:53,685 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T19:13:53,685 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38343 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T19:13:53,687 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1541): 76c4ff0c848e5ffb5ad9989537063596/B is initiating minor compaction (all files) 2024-11-26T19:13:53,687 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1541): 76c4ff0c848e5ffb5ad9989537063596/A is initiating minor compaction (all files) 2024-11-26T19:13:53,688 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 76c4ff0c848e5ffb5ad9989537063596/A in TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. 2024-11-26T19:13:53,688 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 76c4ff0c848e5ffb5ad9989537063596/B in TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. 2024-11-26T19:13:53,688 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/caee9bb5427744818d43a7ad8a612201, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/ad268253061e43b88ec365621129766a, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/84313ca47744468780bcf3b82b5296af] into tmpdir=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp, totalSize=35.2 K 2024-11-26T19:13:53,688 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/a198223cb57a4570b8cb1128e123f979, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/eab18453588b45ddad179336a0d7ccd4, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/1143addfdb464bfbaf1996658cd75569] into tmpdir=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp, totalSize=37.4 K 2024-11-26T19:13:53,690 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting caee9bb5427744818d43a7ad8a612201, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732648430463 2024-11-26T19:13:53,690 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting a198223cb57a4570b8cb1128e123f979, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732648430463 2024-11-26T19:13:53,691 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting ad268253061e43b88ec365621129766a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1732648430786 2024-11-26T19:13:53,692 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting 84313ca47744468780bcf3b82b5296af, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1732648433032 2024-11-26T19:13:53,693 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting eab18453588b45ddad179336a0d7ccd4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1732648430786 2024-11-26T19:13:53,696 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1143addfdb464bfbaf1996658cd75569, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1732648433032 2024-11-26T19:13:53,727 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 76c4ff0c848e5ffb5ad9989537063596#B#compaction#10 average throughput is 0.82 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T19:13:53,727 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 76c4ff0c848e5ffb5ad9989537063596#A#compaction#9 average throughput is 0.82 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T19:13:53,728 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/B/c42911a70e754a3eb876673cbd4e84fa is 50, key is test_row_0/B:col10/1732648433035/Put/seqid=0 2024-11-26T19:13:53,728 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/A/417e2ebbceda48289466048b374abb09 is 50, key is test_row_0/A:col10/1732648433035/Put/seqid=0 2024-11-26T19:13:53,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741846_1022 (size=12104) 2024-11-26T19:13:53,762 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/A/417e2ebbceda48289466048b374abb09 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/417e2ebbceda48289466048b374abb09 2024-11-26T19:13:53,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741847_1023 (size=12104) 2024-11-26T19:13:53,779 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/B/c42911a70e754a3eb876673cbd4e84fa as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/c42911a70e754a3eb876673cbd4e84fa 2024-11-26T19:13:53,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(8855): Flush requested on 76c4ff0c848e5ffb5ad9989537063596 2024-11-26T19:13:53,793 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 76c4ff0c848e5ffb5ad9989537063596/A of 76c4ff0c848e5ffb5ad9989537063596 into 417e2ebbceda48289466048b374abb09(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T19:13:53,793 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 76c4ff0c848e5ffb5ad9989537063596 3/3 column families, dataSize=140.49 KB heapSize=394.62 KB 2024-11-26T19:13:53,793 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 76c4ff0c848e5ffb5ad9989537063596: 2024-11-26T19:13:53,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c4ff0c848e5ffb5ad9989537063596, store=A 2024-11-26T19:13:53,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T19:13:53,794 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c4ff0c848e5ffb5ad9989537063596, store=B 2024-11-26T19:13:53,794 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T19:13:53,794 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c4ff0c848e5ffb5ad9989537063596, store=C 2024-11-26T19:13:53,794 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T19:13:53,796 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., storeName=76c4ff0c848e5ffb5ad9989537063596/A, priority=13, startTime=1732648433672; duration=0sec 2024-11-26T19:13:53,797 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T19:13:53,797 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 76c4ff0c848e5ffb5ad9989537063596:A 2024-11-26T19:13:53,797 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T19:13:53,805 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T19:13:53,805 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1541): 76c4ff0c848e5ffb5ad9989537063596/C is initiating minor compaction (all files) 2024-11-26T19:13:53,805 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 76c4ff0c848e5ffb5ad9989537063596/C in TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. 2024-11-26T19:13:53,805 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/4b5236a5c5d14cd6bf4d3b06b23d2f94, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/1beea36d53264c0cb8b3f76dcac2c531, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/c72b1f40d9a84d0a8c0ce7fbd4efb161] into tmpdir=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp, totalSize=35.2 K 2024-11-26T19:13:53,806 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4b5236a5c5d14cd6bf4d3b06b23d2f94, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732648430463 2024-11-26T19:13:53,808 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1beea36d53264c0cb8b3f76dcac2c531, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1732648430786 2024-11-26T19:13:53,809 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting c72b1f40d9a84d0a8c0ce7fbd4efb161, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1732648433032 2024-11-26T19:13:53,824 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/A/a059261dcd974c5d8eed5b1d29328488 is 50, key is test_row_0/A:col10/1732648433193/Put/seqid=0 2024-11-26T19:13:53,838 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 76c4ff0c848e5ffb5ad9989537063596/B of 76c4ff0c848e5ffb5ad9989537063596 into c42911a70e754a3eb876673cbd4e84fa(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T19:13:53,838 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 76c4ff0c848e5ffb5ad9989537063596: 2024-11-26T19:13:53,839 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., storeName=76c4ff0c848e5ffb5ad9989537063596/B, priority=13, startTime=1732648433680; duration=0sec 2024-11-26T19:13:53,839 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:13:53,839 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 76c4ff0c848e5ffb5ad9989537063596:B 2024-11-26T19:13:53,852 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 76c4ff0c848e5ffb5ad9989537063596#C#compaction#12 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T19:13:53,853 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/C/275b2f0bc7a040b4abb39b9e69c93b2b is 50, key is test_row_0/C:col10/1732648433035/Put/seqid=0 2024-11-26T19:13:53,871 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:13:53,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49132 deadline: 1732648493863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:13:53,873 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:13:53,873 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:53,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49182 deadline: 1732648493866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:13:53,873 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:53,874 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:13:53,874 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:53,874 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:53,874 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:13:53,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741848_1024 (size=12001) 2024-11-26T19:13:53,897 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:13:53,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49210 deadline: 1732648493892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:13:53,898 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:53,899 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:53,899 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:13:53,899 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:13:53,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49154 deadline: 1732648493892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:13:53,900 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:53,900 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:53,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741849_1025 (size=12104) 2024-11-26T19:13:53,900 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:13:53,907 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:13:53,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49198 deadline: 1732648493902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:13:53,909 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:53,909 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:53,909 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:13:53,985 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:13:53,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49182 deadline: 1732648493983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:13:53,986 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:53,986 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:13:53,986 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:53,986 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:13:53,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49132 deadline: 1732648493983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:13:53,987 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:53,988 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:53,988 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:13:54,201 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:13:54,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49182 deadline: 1732648494193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:13:54,202 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:54,202 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:13:54,202 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:54,202 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:13:54,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49132 deadline: 1732648494194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:13:54,203 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:54,203 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:54,203 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:13:54,278 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/A/a059261dcd974c5d8eed5b1d29328488 2024-11-26T19:13:54,300 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/B/2d4a10652e2f46a5ac9b71d9b13c322d is 50, key is test_row_0/B:col10/1732648433193/Put/seqid=0 2024-11-26T19:13:54,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741850_1026 (size=12001) 2024-11-26T19:13:54,314 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/C/275b2f0bc7a040b4abb39b9e69c93b2b as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/275b2f0bc7a040b4abb39b9e69c93b2b 2024-11-26T19:13:54,329 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/B/2d4a10652e2f46a5ac9b71d9b13c322d 2024-11-26T19:13:54,334 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 76c4ff0c848e5ffb5ad9989537063596/C of 76c4ff0c848e5ffb5ad9989537063596 into 275b2f0bc7a040b4abb39b9e69c93b2b(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T19:13:54,334 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 76c4ff0c848e5ffb5ad9989537063596: 2024-11-26T19:13:54,334 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., storeName=76c4ff0c848e5ffb5ad9989537063596/C, priority=13, startTime=1732648433685; duration=0sec 2024-11-26T19:13:54,334 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:13:54,334 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 76c4ff0c848e5ffb5ad9989537063596:C 2024-11-26T19:13:54,346 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/C/3d0d5d31416f496e872554561169b924 is 50, key is test_row_0/C:col10/1732648433193/Put/seqid=0 2024-11-26T19:13:54,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741851_1027 (size=12001) 2024-11-26T19:13:54,364 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.09 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/C/3d0d5d31416f496e872554561169b924 2024-11-26T19:13:54,376 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/A/a059261dcd974c5d8eed5b1d29328488 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/a059261dcd974c5d8eed5b1d29328488 2024-11-26T19:13:54,387 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/a059261dcd974c5d8eed5b1d29328488, entries=150, sequenceid=95, filesize=11.7 K 2024-11-26T19:13:54,389 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/B/2d4a10652e2f46a5ac9b71d9b13c322d as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/2d4a10652e2f46a5ac9b71d9b13c322d 2024-11-26T19:13:54,402 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/2d4a10652e2f46a5ac9b71d9b13c322d, entries=150, sequenceid=95, filesize=11.7 K 2024-11-26T19:13:54,404 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/C/3d0d5d31416f496e872554561169b924 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/3d0d5d31416f496e872554561169b924 2024-11-26T19:13:54,414 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:13:54,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49154 deadline: 1732648494412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:13:54,416 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:13:54,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49210 deadline: 1732648494412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:13:54,417 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:54,417 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:54,417 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:13:54,417 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:54,417 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:54,417 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:13:54,418 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/3d0d5d31416f496e872554561169b924, entries=150, sequenceid=95, filesize=11.7 K 2024-11-26T19:13:54,423 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.49 KB/143865, heapSize ~394.57 KB/404040, currentSize=40.25 KB/41220 for 76c4ff0c848e5ffb5ad9989537063596 in 629ms, sequenceid=95, compaction requested=false 2024-11-26T19:13:54,423 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 76c4ff0c848e5ffb5ad9989537063596: 2024-11-26T19:13:54,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(8855): Flush requested on 76c4ff0c848e5ffb5ad9989537063596 2024-11-26T19:13:54,430 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 76c4ff0c848e5ffb5ad9989537063596 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-26T19:13:54,431 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c4ff0c848e5ffb5ad9989537063596, store=A 2024-11-26T19:13:54,431 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:13:54,431 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c4ff0c848e5ffb5ad9989537063596, store=B 2024-11-26T19:13:54,431 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:13:54,431 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c4ff0c848e5ffb5ad9989537063596, store=C 2024-11-26T19:13:54,431 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:13:54,455 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/A/bce7dd392f5c4c1db67aab71d936efec is 50, key is test_row_0/A:col10/1732648433853/Put/seqid=0 2024-11-26T19:13:54,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741852_1028 (size=12001) 2024-11-26T19:13:54,469 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=107 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/A/bce7dd392f5c4c1db67aab71d936efec 2024-11-26T19:13:54,491 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/B/753c8c618f1f4d4bb636e3d4bb699e98 is 50, key is test_row_0/B:col10/1732648433853/Put/seqid=0 2024-11-26T19:13:54,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741853_1029 (size=12001) 2024-11-26T19:13:54,509 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=107 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/B/753c8c618f1f4d4bb636e3d4bb699e98 2024-11-26T19:13:54,535 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/C/8a83380a02c1473689c9afa4aa69aecc is 50, key is test_row_0/C:col10/1732648433853/Put/seqid=0 2024-11-26T19:13:54,552 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732648434522, max timestamp=1732648434543Num uniques 859; 2024-11-26T19:13:54,552 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-26T19:13:54,555 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732648434522, max timestamp=1732648434546Num uniques 859; 2024-11-26T19:13:54,556 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-26T19:13:54,564 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732648434522, max timestamp=1732648434543Num uniques 859; 2024-11-26T19:13:54,564 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-26T19:13:54,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741854_1030 (size=12001) 2024-11-26T19:13:54,567 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=107 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/C/8a83380a02c1473689c9afa4aa69aecc 2024-11-26T19:13:54,583 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/A/bce7dd392f5c4c1db67aab71d936efec as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/bce7dd392f5c4c1db67aab71d936efec 2024-11-26T19:13:54,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-26T19:13:54,618 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T19:13:54,619 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33811 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T19:13:54,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33811 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-26T19:13:54,623 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/bce7dd392f5c4c1db67aab71d936efec, entries=150, sequenceid=107, filesize=11.7 K 2024-11-26T19:13:54,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-26T19:13:54,624 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T19:13:54,625 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T19:13:54,625 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/B/753c8c618f1f4d4bb636e3d4bb699e98 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/753c8c618f1f4d4bb636e3d4bb699e98 2024-11-26T19:13:54,625 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T19:13:54,639 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/753c8c618f1f4d4bb636e3d4bb699e98, entries=150, sequenceid=107, filesize=11.7 K 2024-11-26T19:13:54,641 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/C/8a83380a02c1473689c9afa4aa69aecc as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/8a83380a02c1473689c9afa4aa69aecc 2024-11-26T19:13:54,658 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/8a83380a02c1473689c9afa4aa69aecc, entries=150, sequenceid=107, filesize=11.7 K 2024-11-26T19:13:54,661 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=93.93 KB/96180 for 76c4ff0c848e5ffb5ad9989537063596 in 232ms, sequenceid=107, compaction requested=true 2024-11-26T19:13:54,661 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 76c4ff0c848e5ffb5ad9989537063596: 2024-11-26T19:13:54,661 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 76c4ff0c848e5ffb5ad9989537063596:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T19:13:54,661 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:13:54,661 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T19:13:54,662 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 76c4ff0c848e5ffb5ad9989537063596:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T19:13:54,662 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:13:54,662 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T19:13:54,662 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 76c4ff0c848e5ffb5ad9989537063596:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T19:13:54,662 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T19:13:54,664 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T19:13:54,664 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1541): 76c4ff0c848e5ffb5ad9989537063596/A is initiating minor compaction (all files) 2024-11-26T19:13:54,664 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T19:13:54,664 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 76c4ff0c848e5ffb5ad9989537063596/A in TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. 2024-11-26T19:13:54,664 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1541): 76c4ff0c848e5ffb5ad9989537063596/B is initiating minor compaction (all files) 2024-11-26T19:13:54,664 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 76c4ff0c848e5ffb5ad9989537063596/B in TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. 2024-11-26T19:13:54,664 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/417e2ebbceda48289466048b374abb09, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/a059261dcd974c5d8eed5b1d29328488, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/bce7dd392f5c4c1db67aab71d936efec] into tmpdir=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp, totalSize=35.3 K 2024-11-26T19:13:54,665 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/c42911a70e754a3eb876673cbd4e84fa, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/2d4a10652e2f46a5ac9b71d9b13c322d, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/753c8c618f1f4d4bb636e3d4bb699e98] into tmpdir=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp, totalSize=35.3 K 2024-11-26T19:13:54,665 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting 417e2ebbceda48289466048b374abb09, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1732648433032 2024-11-26T19:13:54,665 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting c42911a70e754a3eb876673cbd4e84fa, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1732648433032 2024-11-26T19:13:54,666 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting a059261dcd974c5d8eed5b1d29328488, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732648433193 2024-11-26T19:13:54,666 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting 2d4a10652e2f46a5ac9b71d9b13c322d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732648433193 2024-11-26T19:13:54,667 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting bce7dd392f5c4c1db67aab71d936efec, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1732648433853 2024-11-26T19:13:54,667 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting 753c8c618f1f4d4bb636e3d4bb699e98, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1732648433853 2024-11-26T19:13:54,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(8855): Flush requested on 76c4ff0c848e5ffb5ad9989537063596 2024-11-26T19:13:54,671 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 76c4ff0c848e5ffb5ad9989537063596 3/3 column families, dataSize=107.34 KB heapSize=305.30 KB 2024-11-26T19:13:54,671 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c4ff0c848e5ffb5ad9989537063596, store=A 2024-11-26T19:13:54,671 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T19:13:54,672 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c4ff0c848e5ffb5ad9989537063596, store=B 2024-11-26T19:13:54,672 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T19:13:54,672 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c4ff0c848e5ffb5ad9989537063596, store=C 2024-11-26T19:13:54,672 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T19:13:54,687 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/A/35f3765d522a4a5cb13ee27baeea2c1b is 50, key is test_row_0/A:col10/1732648434668/Put/seqid=0 2024-11-26T19:13:54,691 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 76c4ff0c848e5ffb5ad9989537063596#A#compaction#19 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T19:13:54,692 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/A/03c3e8b4bbb54f09acf353e917df4e86 is 50, key is test_row_0/A:col10/1732648433853/Put/seqid=0 2024-11-26T19:13:54,701 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 76c4ff0c848e5ffb5ad9989537063596#B#compaction#20 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T19:13:54,702 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/B/af438f13b1204e9e8acfc13f307c89d4 is 50, key is test_row_0/B:col10/1732648433853/Put/seqid=0 2024-11-26T19:13:54,725 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:13:54,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49198 deadline: 1732648494718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:13:54,727 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:54,727 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:54,727 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:13:54,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741855_1031 (size=12151) 2024-11-26T19:13:54,731 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=140 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/A/35f3765d522a4a5cb13ee27baeea2c1b 2024-11-26T19:13:54,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-26T19:13:54,736 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:13:54,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49132 deadline: 1732648494722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:13:54,737 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:54,737 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:54,737 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:13:54,743 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:13:54,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49182 deadline: 1732648494729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:13:54,745 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:54,745 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:54,745 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:13:54,747 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741856_1032 (size=12207) 2024-11-26T19:13:54,759 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/A/03c3e8b4bbb54f09acf353e917df4e86 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/03c3e8b4bbb54f09acf353e917df4e86 2024-11-26T19:13:54,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741857_1033 (size=12207) 2024-11-26T19:13:54,776 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/B/a8898f8302174564861f2109f1015483 is 50, key is test_row_0/B:col10/1732648434668/Put/seqid=0 2024-11-26T19:13:54,776 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 76c4ff0c848e5ffb5ad9989537063596/A of 76c4ff0c848e5ffb5ad9989537063596 into 03c3e8b4bbb54f09acf353e917df4e86(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T19:13:54,778 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 76c4ff0c848e5ffb5ad9989537063596: 2024-11-26T19:13:54,778 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., storeName=76c4ff0c848e5ffb5ad9989537063596/A, priority=13, startTime=1732648434661; duration=0sec 2024-11-26T19:13:54,778 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T19:13:54,778 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 76c4ff0c848e5ffb5ad9989537063596:A 2024-11-26T19:13:54,778 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T19:13:54,781 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/B/af438f13b1204e9e8acfc13f307c89d4 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/af438f13b1204e9e8acfc13f307c89d4 2024-11-26T19:13:54,785 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-26T19:13:54,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. 2024-11-26T19:13:54,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. as already flushing 2024-11-26T19:13:54,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. 2024-11-26T19:13:54,786 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:13:54,786 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T19:13:54,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:13:54,786 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1541): 76c4ff0c848e5ffb5ad9989537063596/C is initiating minor compaction (all files) 2024-11-26T19:13:54,786 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 76c4ff0c848e5ffb5ad9989537063596/C in TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. 2024-11-26T19:13:54,786 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/275b2f0bc7a040b4abb39b9e69c93b2b, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/3d0d5d31416f496e872554561169b924, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/8a83380a02c1473689c9afa4aa69aecc] into tmpdir=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp, totalSize=35.3 K 2024-11-26T19:13:54,787 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting 275b2f0bc7a040b4abb39b9e69c93b2b, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1732648433032 2024-11-26T19:13:54,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:13:54,788 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3d0d5d31416f496e872554561169b924, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732648433193 2024-11-26T19:13:54,789 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8a83380a02c1473689c9afa4aa69aecc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1732648433853 2024-11-26T19:13:54,797 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 76c4ff0c848e5ffb5ad9989537063596/B of 76c4ff0c848e5ffb5ad9989537063596 into af438f13b1204e9e8acfc13f307c89d4(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T19:13:54,797 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 76c4ff0c848e5ffb5ad9989537063596: 2024-11-26T19:13:54,797 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., storeName=76c4ff0c848e5ffb5ad9989537063596/B, priority=13, startTime=1732648434661; duration=0sec 2024-11-26T19:13:54,798 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:13:54,798 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 76c4ff0c848e5ffb5ad9989537063596:B 2024-11-26T19:13:54,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741858_1034 (size=12151) 2024-11-26T19:13:54,804 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 76c4ff0c848e5ffb5ad9989537063596#C#compaction#22 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T19:13:54,804 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/C/5ccecf4f55814107ab8d667473719261 is 50, key is test_row_0/C:col10/1732648433853/Put/seqid=0 2024-11-26T19:13:54,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741859_1035 (size=12207) 2024-11-26T19:13:54,832 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/C/5ccecf4f55814107ab8d667473719261 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/5ccecf4f55814107ab8d667473719261 2024-11-26T19:13:54,833 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:13:54,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49198 deadline: 1732648494832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:13:54,834 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:54,834 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:54,834 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:13:54,845 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 76c4ff0c848e5ffb5ad9989537063596/C of 76c4ff0c848e5ffb5ad9989537063596 into 5ccecf4f55814107ab8d667473719261(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T19:13:54,845 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 76c4ff0c848e5ffb5ad9989537063596: 2024-11-26T19:13:54,845 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., storeName=76c4ff0c848e5ffb5ad9989537063596/C, priority=13, startTime=1732648434662; duration=0sec 2024-11-26T19:13:54,845 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:13:54,845 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 76c4ff0c848e5ffb5ad9989537063596:C 2024-11-26T19:13:54,848 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:13:54,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49132 deadline: 1732648494842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:13:54,851 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:54,851 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:54,851 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:13:54,857 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:13:54,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49182 deadline: 1732648494852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:13:54,858 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:54,858 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:54,858 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:13:54,940 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-26T19:13:54,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. 2024-11-26T19:13:54,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. as already flushing 2024-11-26T19:13:54,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. 2024-11-26T19:13:54,941 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:13:54,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:13:54,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-26T19:13:54,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:13:55,046 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:13:55,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49198 deadline: 1732648495043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:13:55,047 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:55,047 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:55,047 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:13:55,063 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:13:55,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49132 deadline: 1732648495062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:13:55,064 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:55,064 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:55,064 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:13:55,067 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:13:55,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49182 deadline: 1732648495063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:13:55,068 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:55,068 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:55,068 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:13:55,095 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-26T19:13:55,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. 2024-11-26T19:13:55,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. as already flushing 2024-11-26T19:13:55,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. 2024-11-26T19:13:55,096 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:13:55,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:13:55,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:13:55,204 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=140 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/B/a8898f8302174564861f2109f1015483 2024-11-26T19:13:55,225 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/C/f1b3d656676a42feba4d88e1c212d3ba is 50, key is test_row_0/C:col10/1732648434668/Put/seqid=0 2024-11-26T19:13:55,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741860_1036 (size=12151) 2024-11-26T19:13:55,252 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-26T19:13:55,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. 2024-11-26T19:13:55,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. as already flushing 2024-11-26T19:13:55,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-26T19:13:55,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. 2024-11-26T19:13:55,253 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:13:55,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:13:55,254 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=140 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/C/f1b3d656676a42feba4d88e1c212d3ba 2024-11-26T19:13:55,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:13:55,264 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/A/35f3765d522a4a5cb13ee27baeea2c1b as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/35f3765d522a4a5cb13ee27baeea2c1b 2024-11-26T19:13:55,273 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/35f3765d522a4a5cb13ee27baeea2c1b, entries=150, sequenceid=140, filesize=11.9 K 2024-11-26T19:13:55,274 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/B/a8898f8302174564861f2109f1015483 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/a8898f8302174564861f2109f1015483 2024-11-26T19:13:55,283 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/a8898f8302174564861f2109f1015483, entries=150, sequenceid=140, filesize=11.9 K 2024-11-26T19:13:55,285 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/C/f1b3d656676a42feba4d88e1c212d3ba as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/f1b3d656676a42feba4d88e1c212d3ba 2024-11-26T19:13:55,293 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/f1b3d656676a42feba4d88e1c212d3ba, entries=150, sequenceid=140, filesize=11.9 K 2024-11-26T19:13:55,295 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~305.25 KB/312576, currentSize=80.51 KB/82440 for 76c4ff0c848e5ffb5ad9989537063596 in 623ms, sequenceid=140, compaction requested=false 2024-11-26T19:13:55,295 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 76c4ff0c848e5ffb5ad9989537063596: 2024-11-26T19:13:55,356 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 76c4ff0c848e5ffb5ad9989537063596 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-26T19:13:55,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(8855): Flush requested on 76c4ff0c848e5ffb5ad9989537063596 2024-11-26T19:13:55,356 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c4ff0c848e5ffb5ad9989537063596, store=A 2024-11-26T19:13:55,356 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:13:55,356 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c4ff0c848e5ffb5ad9989537063596, store=B 2024-11-26T19:13:55,357 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:13:55,357 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c4ff0c848e5ffb5ad9989537063596, store=C 2024-11-26T19:13:55,357 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:13:55,366 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/A/88f933f70dd24dca965172f1945c0a78 is 50, key is test_row_0/A:col10/1732648434710/Put/seqid=0 2024-11-26T19:13:55,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741861_1037 (size=14541) 2024-11-26T19:13:55,395 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=159 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/A/88f933f70dd24dca965172f1945c0a78 2024-11-26T19:13:55,407 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-26T19:13:55,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. 2024-11-26T19:13:55,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. as already flushing 2024-11-26T19:13:55,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. 2024-11-26T19:13:55,408 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:13:55,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:13:55,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:13:55,420 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/B/8963dd6b3d994966883672717578324d is 50, key is test_row_0/B:col10/1732648434710/Put/seqid=0 2024-11-26T19:13:55,423 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:13:55,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49132 deadline: 1732648495418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:13:55,424 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:55,424 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:55,424 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:13:55,429 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:13:55,430 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:13:55,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49198 deadline: 1732648495425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:13:55,431 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:55,431 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:55,431 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:13:55,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49182 deadline: 1732648495424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:13:55,433 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:55,433 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:55,433 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:13:55,436 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:13:55,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49210 deadline: 1732648495432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:13:55,438 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:13:55,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49154 deadline: 1732648495433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:13:55,441 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to fe76acf732b2,42785,1732648423970 for 'test_row_2' in 76c4ff0c848e5ffb5ad9989537063596 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2185 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T19:13:55,442 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:55,442 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:55,443 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:13:55,444 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to fe76acf732b2,42785,1732648423970 for 'test_row_0' in 76c4ff0c848e5ffb5ad9989537063596 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2185 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T19:13:55,445 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:55,445 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:55,445 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:13:55,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741862_1038 (size=12151) 2024-11-26T19:13:55,462 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=159 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/B/8963dd6b3d994966883672717578324d 2024-11-26T19:13:55,487 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/C/65a9b245a4ee432d81cf41a6ab896966 is 50, key is test_row_0/C:col10/1732648434710/Put/seqid=0 2024-11-26T19:13:55,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741863_1039 (size=12151) 2024-11-26T19:13:55,537 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:13:55,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49132 deadline: 1732648495532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:13:55,539 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:55,539 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:55,539 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:13:55,543 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:13:55,543 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:13:55,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49182 deadline: 1732648495542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:13:55,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49198 deadline: 1732648495542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:13:55,544 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:55,544 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:55,544 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:13:55,545 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:55,545 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:55,545 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:13:55,563 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-26T19:13:55,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. 2024-11-26T19:13:55,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. as already flushing 2024-11-26T19:13:55,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. 2024-11-26T19:13:55,564 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:13:55,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:13:55,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:13:55,718 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-26T19:13:55,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. 2024-11-26T19:13:55,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. as already flushing 2024-11-26T19:13:55,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. 2024-11-26T19:13:55,719 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:13:55,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:13:55,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:13:55,746 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:13:55,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49132 deadline: 1732648495742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:13:55,747 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:55,748 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:55,748 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:13:55,753 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:13:55,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49182 deadline: 1732648495752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:13:55,754 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:55,754 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:55,754 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:13:55,754 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:13:55,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49198 deadline: 1732648495752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:13:55,754 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:55,755 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:55,755 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:13:55,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-26T19:13:55,873 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-26T19:13:55,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. 2024-11-26T19:13:55,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. as already flushing 2024-11-26T19:13:55,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. 2024-11-26T19:13:55,873 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:13:55,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:13:55,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:13:55,907 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=159 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/C/65a9b245a4ee432d81cf41a6ab896966 2024-11-26T19:13:55,917 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/A/88f933f70dd24dca965172f1945c0a78 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/88f933f70dd24dca965172f1945c0a78 2024-11-26T19:13:55,926 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/88f933f70dd24dca965172f1945c0a78, entries=200, sequenceid=159, filesize=14.2 K 2024-11-26T19:13:55,927 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/B/8963dd6b3d994966883672717578324d as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/8963dd6b3d994966883672717578324d 2024-11-26T19:13:55,937 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/8963dd6b3d994966883672717578324d, entries=150, sequenceid=159, filesize=11.9 K 2024-11-26T19:13:55,939 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/C/65a9b245a4ee432d81cf41a6ab896966 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/65a9b245a4ee432d81cf41a6ab896966 2024-11-26T19:13:55,968 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/65a9b245a4ee432d81cf41a6ab896966, entries=150, sequenceid=159, filesize=11.9 K 2024-11-26T19:13:55,970 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=93.93 KB/96180 for 76c4ff0c848e5ffb5ad9989537063596 in 614ms, sequenceid=159, compaction requested=true 2024-11-26T19:13:55,970 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 76c4ff0c848e5ffb5ad9989537063596: 2024-11-26T19:13:55,970 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T19:13:55,970 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 76c4ff0c848e5ffb5ad9989537063596:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T19:13:55,971 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:13:55,971 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 76c4ff0c848e5ffb5ad9989537063596:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T19:13:55,971 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:13:55,971 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 76c4ff0c848e5ffb5ad9989537063596:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T19:13:55,971 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T19:13:55,971 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T19:13:55,973 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38899 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T19:13:55,973 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1541): 76c4ff0c848e5ffb5ad9989537063596/A is initiating minor compaction (all files) 2024-11-26T19:13:55,973 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 76c4ff0c848e5ffb5ad9989537063596/A in TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. 2024-11-26T19:13:55,973 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/03c3e8b4bbb54f09acf353e917df4e86, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/35f3765d522a4a5cb13ee27baeea2c1b, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/88f933f70dd24dca965172f1945c0a78] into tmpdir=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp, totalSize=38.0 K 2024-11-26T19:13:55,973 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36509 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T19:13:55,974 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1541): 76c4ff0c848e5ffb5ad9989537063596/B is initiating minor compaction (all files) 2024-11-26T19:13:55,974 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 76c4ff0c848e5ffb5ad9989537063596/B in TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. 2024-11-26T19:13:55,974 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/af438f13b1204e9e8acfc13f307c89d4, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/a8898f8302174564861f2109f1015483, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/8963dd6b3d994966883672717578324d] into tmpdir=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp, totalSize=35.7 K 2024-11-26T19:13:55,974 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting af438f13b1204e9e8acfc13f307c89d4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1732648433853 2024-11-26T19:13:55,975 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting 03c3e8b4bbb54f09acf353e917df4e86, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1732648433853 2024-11-26T19:13:55,975 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting a8898f8302174564861f2109f1015483, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1732648434653 2024-11-26T19:13:55,975 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting 35f3765d522a4a5cb13ee27baeea2c1b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1732648434653 2024-11-26T19:13:55,976 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8963dd6b3d994966883672717578324d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1732648434710 2024-11-26T19:13:55,977 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting 88f933f70dd24dca965172f1945c0a78, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1732648434710 2024-11-26T19:13:55,994 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 76c4ff0c848e5ffb5ad9989537063596#B#compaction#27 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T19:13:55,995 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/B/c4a0c5cf1f9a4ffcb4b3afd52bb65774 is 50, key is test_row_0/B:col10/1732648434710/Put/seqid=0 2024-11-26T19:13:56,001 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 76c4ff0c848e5ffb5ad9989537063596#A#compaction#28 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T19:13:56,002 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/A/1e60651003db49adad7dbf608e7764ce is 50, key is test_row_0/A:col10/1732648434710/Put/seqid=0 2024-11-26T19:13:56,011 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741864_1040 (size=12459) 2024-11-26T19:13:56,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741865_1041 (size=12459) 2024-11-26T19:13:56,024 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/B/c4a0c5cf1f9a4ffcb4b3afd52bb65774 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/c4a0c5cf1f9a4ffcb4b3afd52bb65774 2024-11-26T19:13:56,027 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-26T19:13:56,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. 2024-11-26T19:13:56,028 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing 76c4ff0c848e5ffb5ad9989537063596 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-26T19:13:56,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c4ff0c848e5ffb5ad9989537063596, store=A 2024-11-26T19:13:56,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:13:56,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c4ff0c848e5ffb5ad9989537063596, store=B 2024-11-26T19:13:56,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:13:56,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c4ff0c848e5ffb5ad9989537063596, store=C 2024-11-26T19:13:56,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:13:56,039 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 76c4ff0c848e5ffb5ad9989537063596/B of 76c4ff0c848e5ffb5ad9989537063596 into c4a0c5cf1f9a4ffcb4b3afd52bb65774(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T19:13:56,039 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 76c4ff0c848e5ffb5ad9989537063596: 2024-11-26T19:13:56,039 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., storeName=76c4ff0c848e5ffb5ad9989537063596/B, priority=13, startTime=1732648435971; duration=0sec 2024-11-26T19:13:56,039 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T19:13:56,040 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 76c4ff0c848e5ffb5ad9989537063596:B 2024-11-26T19:13:56,040 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T19:13:56,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/A/de1be7379cca41919698005e01bc443d is 50, key is test_row_0/A:col10/1732648435418/Put/seqid=0 2024-11-26T19:13:56,043 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36509 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T19:13:56,043 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1541): 76c4ff0c848e5ffb5ad9989537063596/C is initiating minor compaction (all files) 2024-11-26T19:13:56,043 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 76c4ff0c848e5ffb5ad9989537063596/C in TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. 2024-11-26T19:13:56,044 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/5ccecf4f55814107ab8d667473719261, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/f1b3d656676a42feba4d88e1c212d3ba, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/65a9b245a4ee432d81cf41a6ab896966] into tmpdir=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp, totalSize=35.7 K 2024-11-26T19:13:56,044 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5ccecf4f55814107ab8d667473719261, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1732648433853 2024-11-26T19:13:56,045 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting f1b3d656676a42feba4d88e1c212d3ba, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1732648434653 2024-11-26T19:13:56,046 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting 65a9b245a4ee432d81cf41a6ab896966, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1732648434710 2024-11-26T19:13:56,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741866_1042 (size=12151) 2024-11-26T19:13:56,059 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=176 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/A/de1be7379cca41919698005e01bc443d 2024-11-26T19:13:56,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(8855): Flush requested on 76c4ff0c848e5ffb5ad9989537063596 2024-11-26T19:13:56,067 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 76c4ff0c848e5ffb5ad9989537063596#C#compaction#30 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T19:13:56,068 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/C/966e3641f55f4a57afe91067d2a75499 is 50, key is test_row_0/C:col10/1732648434710/Put/seqid=0 2024-11-26T19:13:56,068 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. as already flushing 2024-11-26T19:13:56,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/B/e340a0fc862e4e6899c9ba3cce88c90d is 50, key is test_row_0/B:col10/1732648435418/Put/seqid=0 2024-11-26T19:13:56,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741867_1043 (size=12459) 2024-11-26T19:13:56,106 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:13:56,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49198 deadline: 1732648496099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:13:56,107 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:56,107 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:56,107 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:13:56,108 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:13:56,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49182 deadline: 1732648496101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:13:56,109 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:56,109 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:13:56,109 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:56,109 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:13:56,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49132 deadline: 1732648496101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:13:56,110 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:56,110 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:56,110 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:13:56,117 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/C/966e3641f55f4a57afe91067d2a75499 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/966e3641f55f4a57afe91067d2a75499 2024-11-26T19:13:56,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741868_1044 (size=12151) 2024-11-26T19:13:56,142 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 76c4ff0c848e5ffb5ad9989537063596/C of 76c4ff0c848e5ffb5ad9989537063596 into 966e3641f55f4a57afe91067d2a75499(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T19:13:56,142 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 76c4ff0c848e5ffb5ad9989537063596: 2024-11-26T19:13:56,142 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., storeName=76c4ff0c848e5ffb5ad9989537063596/C, priority=13, startTime=1732648435971; duration=0sec 2024-11-26T19:13:56,142 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:13:56,142 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 76c4ff0c848e5ffb5ad9989537063596:C 2024-11-26T19:13:56,212 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:13:56,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49198 deadline: 1732648496212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:13:56,213 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:56,213 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:13:56,213 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:56,214 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:13:56,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49182 deadline: 1732648496212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:13:56,214 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:56,214 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:56,214 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:13:56,214 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:13:56,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49132 deadline: 1732648496212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:13:56,215 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:56,215 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:56,215 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:13:56,424 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:13:56,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49198 deadline: 1732648496422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:13:56,425 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:56,425 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:13:56,425 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:56,425 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:13:56,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49182 deadline: 1732648496422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:13:56,426 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:56,426 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:56,426 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:13:56,429 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:13:56,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49132 deadline: 1732648496427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:13:56,429 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/A/1e60651003db49adad7dbf608e7764ce as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/1e60651003db49adad7dbf608e7764ce 2024-11-26T19:13:56,429 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:56,430 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:56,430 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:13:56,441 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 76c4ff0c848e5ffb5ad9989537063596/A of 76c4ff0c848e5ffb5ad9989537063596 into 1e60651003db49adad7dbf608e7764ce(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T19:13:56,441 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 76c4ff0c848e5ffb5ad9989537063596: 2024-11-26T19:13:56,442 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., storeName=76c4ff0c848e5ffb5ad9989537063596/A, priority=13, startTime=1732648435970; duration=0sec 2024-11-26T19:13:56,442 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:13:56,442 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 76c4ff0c848e5ffb5ad9989537063596:A 2024-11-26T19:13:56,536 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=176 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/B/e340a0fc862e4e6899c9ba3cce88c90d 2024-11-26T19:13:56,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/C/04ac804bd0d64101b082009969bb3647 is 50, key is test_row_0/C:col10/1732648435418/Put/seqid=0 2024-11-26T19:13:56,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741869_1045 (size=12151) 2024-11-26T19:13:56,587 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=176 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/C/04ac804bd0d64101b082009969bb3647 2024-11-26T19:13:56,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/A/de1be7379cca41919698005e01bc443d as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/de1be7379cca41919698005e01bc443d 2024-11-26T19:13:56,621 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/de1be7379cca41919698005e01bc443d, entries=150, sequenceid=176, filesize=11.9 K 2024-11-26T19:13:56,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/B/e340a0fc862e4e6899c9ba3cce88c90d as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/e340a0fc862e4e6899c9ba3cce88c90d 2024-11-26T19:13:56,634 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/e340a0fc862e4e6899c9ba3cce88c90d, entries=150, sequenceid=176, filesize=11.9 K 2024-11-26T19:13:56,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/C/04ac804bd0d64101b082009969bb3647 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/04ac804bd0d64101b082009969bb3647 2024-11-26T19:13:56,649 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/04ac804bd0d64101b082009969bb3647, entries=150, sequenceid=176, filesize=11.9 K 2024-11-26T19:13:56,656 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=87.22 KB/89310 for 76c4ff0c848e5ffb5ad9989537063596 in 628ms, sequenceid=176, compaction requested=false 2024-11-26T19:13:56,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for 76c4ff0c848e5ffb5ad9989537063596: 2024-11-26T19:13:56,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. 2024-11-26T19:13:56,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-26T19:13:56,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-26T19:13:56,662 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-26T19:13:56,663 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0340 sec 2024-11-26T19:13:56,666 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 2.0450 sec 2024-11-26T19:13:56,738 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 76c4ff0c848e5ffb5ad9989537063596 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-26T19:13:56,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(8855): Flush requested on 76c4ff0c848e5ffb5ad9989537063596 2024-11-26T19:13:56,741 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c4ff0c848e5ffb5ad9989537063596, store=A 2024-11-26T19:13:56,741 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:13:56,741 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c4ff0c848e5ffb5ad9989537063596, store=B 2024-11-26T19:13:56,741 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:13:56,741 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c4ff0c848e5ffb5ad9989537063596, store=C 2024-11-26T19:13:56,741 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:13:56,753 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/A/128c2dc5a4704ea6ac55fce0184eb846 is 50, key is test_row_0/A:col10/1732648436099/Put/seqid=0 2024-11-26T19:13:56,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741870_1046 (size=12151) 2024-11-26T19:13:56,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-26T19:13:56,773 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T19:13:56,773 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T19:13:56,774 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/A/128c2dc5a4704ea6ac55fce0184eb846 2024-11-26T19:13:56,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-26T19:13:56,777 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T19:13:56,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-26T19:13:56,779 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T19:13:56,780 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T19:13:56,785 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:13:56,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49182 deadline: 1732648496782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:13:56,786 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:56,787 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:56,787 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:13:56,791 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:13:56,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49198 deadline: 1732648496786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:13:56,792 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:56,792 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:56,792 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:13:56,792 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:13:56,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49132 deadline: 1732648496786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:13:56,792 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:56,793 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:56,793 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:13:56,794 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/B/674882adcdff4f50bb75443483598cef is 50, key is test_row_0/B:col10/1732648436099/Put/seqid=0 2024-11-26T19:13:56,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741871_1047 (size=12151) 2024-11-26T19:13:56,807 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/B/674882adcdff4f50bb75443483598cef 2024-11-26T19:13:56,821 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/C/918fb8fd1f8841b2886ef357573c1940 is 50, key is test_row_0/C:col10/1732648436099/Put/seqid=0 2024-11-26T19:13:56,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741872_1048 (size=12151) 2024-11-26T19:13:56,842 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/C/918fb8fd1f8841b2886ef357573c1940 2024-11-26T19:13:56,859 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/A/128c2dc5a4704ea6ac55fce0184eb846 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/128c2dc5a4704ea6ac55fce0184eb846 2024-11-26T19:13:56,881 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/128c2dc5a4704ea6ac55fce0184eb846, entries=150, sequenceid=198, filesize=11.9 K 2024-11-26T19:13:56,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-26T19:13:56,885 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/B/674882adcdff4f50bb75443483598cef as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/674882adcdff4f50bb75443483598cef 2024-11-26T19:13:56,892 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:13:56,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49182 deadline: 1732648496891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:13:56,893 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:56,893 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:56,893 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:13:56,911 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:13:56,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49132 deadline: 1732648496902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:13:56,911 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:13:56,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49198 deadline: 1732648496902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:13:56,911 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:56,912 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:56,912 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:13:56,912 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:56,912 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:56,912 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:13:56,912 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/674882adcdff4f50bb75443483598cef, entries=150, sequenceid=198, filesize=11.9 K 2024-11-26T19:13:56,917 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/C/918fb8fd1f8841b2886ef357573c1940 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/918fb8fd1f8841b2886ef357573c1940 2024-11-26T19:13:56,925 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/918fb8fd1f8841b2886ef357573c1940, entries=150, sequenceid=198, filesize=11.9 K 2024-11-26T19:13:56,928 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=73.80 KB/75570 for 76c4ff0c848e5ffb5ad9989537063596 in 190ms, sequenceid=198, compaction requested=true 2024-11-26T19:13:56,928 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 76c4ff0c848e5ffb5ad9989537063596: 2024-11-26T19:13:56,928 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 76c4ff0c848e5ffb5ad9989537063596:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T19:13:56,929 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:13:56,929 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T19:13:56,929 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T19:13:56,929 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 76c4ff0c848e5ffb5ad9989537063596:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T19:13:56,929 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:13:56,930 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 76c4ff0c848e5ffb5ad9989537063596:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T19:13:56,930 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T19:13:56,934 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T19:13:56,934 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1541): 76c4ff0c848e5ffb5ad9989537063596/A is initiating minor compaction (all files) 2024-11-26T19:13:56,934 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 76c4ff0c848e5ffb5ad9989537063596/A in TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. 2024-11-26T19:13:56,934 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/1e60651003db49adad7dbf608e7764ce, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/de1be7379cca41919698005e01bc443d, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/128c2dc5a4704ea6ac55fce0184eb846] into tmpdir=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp, totalSize=35.9 K 2024-11-26T19:13:56,934 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T19:13:56,935 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1541): 76c4ff0c848e5ffb5ad9989537063596/B is initiating minor compaction (all files) 2024-11-26T19:13:56,935 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 76c4ff0c848e5ffb5ad9989537063596/B in TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. 2024-11-26T19:13:56,935 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1e60651003db49adad7dbf608e7764ce, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1732648434710 2024-11-26T19:13:56,936 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/c4a0c5cf1f9a4ffcb4b3afd52bb65774, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/e340a0fc862e4e6899c9ba3cce88c90d, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/674882adcdff4f50bb75443483598cef] into tmpdir=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp, totalSize=35.9 K 2024-11-26T19:13:56,936 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting de1be7379cca41919698005e01bc443d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1732648435416 2024-11-26T19:13:56,937 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting c4a0c5cf1f9a4ffcb4b3afd52bb65774, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1732648434710 2024-11-26T19:13:56,937 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-26T19:13:56,937 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting 128c2dc5a4704ea6ac55fce0184eb846, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1732648436099 2024-11-26T19:13:56,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. 2024-11-26T19:13:56,938 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing 76c4ff0c848e5ffb5ad9989537063596 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-26T19:13:56,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c4ff0c848e5ffb5ad9989537063596, store=A 2024-11-26T19:13:56,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:13:56,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c4ff0c848e5ffb5ad9989537063596, store=B 2024-11-26T19:13:56,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:13:56,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c4ff0c848e5ffb5ad9989537063596, store=C 2024-11-26T19:13:56,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:13:56,939 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting e340a0fc862e4e6899c9ba3cce88c90d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1732648435416 2024-11-26T19:13:56,940 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting 674882adcdff4f50bb75443483598cef, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1732648436099 2024-11-26T19:13:56,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/A/6c9a6be2cdd34d3e80c6d012d0bddb33 is 50, key is test_row_0/A:col10/1732648436782/Put/seqid=0 2024-11-26T19:13:56,979 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 76c4ff0c848e5ffb5ad9989537063596#A#compaction#37 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T19:13:56,980 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/A/139e2b3b8af743bfbecc48835a1357dc is 50, key is test_row_0/A:col10/1732648436099/Put/seqid=0 2024-11-26T19:13:56,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741873_1049 (size=12151) 2024-11-26T19:13:57,006 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 76c4ff0c848e5ffb5ad9989537063596#B#compaction#38 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T19:13:57,006 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/B/7b79c14e7270430095e6008dcaeece65 is 50, key is test_row_0/B:col10/1732648436099/Put/seqid=0 2024-11-26T19:13:57,063 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741874_1050 (size=12561) 2024-11-26T19:13:57,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741875_1051 (size=12561) 2024-11-26T19:13:57,077 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/A/139e2b3b8af743bfbecc48835a1357dc as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/139e2b3b8af743bfbecc48835a1357dc 2024-11-26T19:13:57,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-26T19:13:57,094 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 76c4ff0c848e5ffb5ad9989537063596/A of 76c4ff0c848e5ffb5ad9989537063596 into 139e2b3b8af743bfbecc48835a1357dc(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T19:13:57,094 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 76c4ff0c848e5ffb5ad9989537063596: 2024-11-26T19:13:57,094 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., storeName=76c4ff0c848e5ffb5ad9989537063596/A, priority=13, startTime=1732648436928; duration=0sec 2024-11-26T19:13:57,095 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T19:13:57,095 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 76c4ff0c848e5ffb5ad9989537063596:A 2024-11-26T19:13:57,095 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T19:13:57,106 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T19:13:57,106 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1541): 76c4ff0c848e5ffb5ad9989537063596/C is initiating minor compaction (all files) 2024-11-26T19:13:57,106 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 76c4ff0c848e5ffb5ad9989537063596/C in TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. 2024-11-26T19:13:57,106 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/966e3641f55f4a57afe91067d2a75499, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/04ac804bd0d64101b082009969bb3647, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/918fb8fd1f8841b2886ef357573c1940] into tmpdir=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp, totalSize=35.9 K 2024-11-26T19:13:57,108 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/B/7b79c14e7270430095e6008dcaeece65 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/7b79c14e7270430095e6008dcaeece65 2024-11-26T19:13:57,108 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting 966e3641f55f4a57afe91067d2a75499, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1732648434710 2024-11-26T19:13:57,109 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. as already flushing 2024-11-26T19:13:57,109 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting 04ac804bd0d64101b082009969bb3647, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1732648435416 2024-11-26T19:13:57,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(8855): Flush requested on 76c4ff0c848e5ffb5ad9989537063596 2024-11-26T19:13:57,116 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting 918fb8fd1f8841b2886ef357573c1940, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1732648436099 2024-11-26T19:13:57,124 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 76c4ff0c848e5ffb5ad9989537063596/B of 76c4ff0c848e5ffb5ad9989537063596 into 7b79c14e7270430095e6008dcaeece65(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T19:13:57,124 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 76c4ff0c848e5ffb5ad9989537063596: 2024-11-26T19:13:57,124 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., storeName=76c4ff0c848e5ffb5ad9989537063596/B, priority=13, startTime=1732648436929; duration=0sec 2024-11-26T19:13:57,124 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:13:57,124 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 76c4ff0c848e5ffb5ad9989537063596:B 2024-11-26T19:13:57,134 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 76c4ff0c848e5ffb5ad9989537063596#C#compaction#39 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T19:13:57,135 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/C/ca33c2a4ec0749d19d651b83f18f98f5 is 50, key is test_row_0/C:col10/1732648436099/Put/seqid=0 2024-11-26T19:13:57,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741876_1052 (size=12561) 2024-11-26T19:13:57,174 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:13:57,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49132 deadline: 1732648497170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:13:57,175 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:57,175 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:57,175 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:13:57,176 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/C/ca33c2a4ec0749d19d651b83f18f98f5 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/ca33c2a4ec0749d19d651b83f18f98f5 2024-11-26T19:13:57,181 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:13:57,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49182 deadline: 1732648497174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:13:57,181 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:57,182 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:57,181 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:13:57,182 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:13:57,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49198 deadline: 1732648497174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:13:57,182 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:57,182 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:57,182 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:13:57,186 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 76c4ff0c848e5ffb5ad9989537063596/C of 76c4ff0c848e5ffb5ad9989537063596 into ca33c2a4ec0749d19d651b83f18f98f5(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T19:13:57,186 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 76c4ff0c848e5ffb5ad9989537063596: 2024-11-26T19:13:57,186 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., storeName=76c4ff0c848e5ffb5ad9989537063596/C, priority=13, startTime=1732648436929; duration=0sec 2024-11-26T19:13:57,186 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:13:57,186 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 76c4ff0c848e5ffb5ad9989537063596:C 2024-11-26T19:13:57,284 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:13:57,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49132 deadline: 1732648497282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:13:57,285 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:57,285 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:57,286 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:13:57,295 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:13:57,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49182 deadline: 1732648497292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:13:57,296 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:57,296 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:57,296 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:13:57,296 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:13:57,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49198 deadline: 1732648497292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:13:57,297 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:57,297 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:57,297 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:13:57,398 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/A/6c9a6be2cdd34d3e80c6d012d0bddb33 2024-11-26T19:13:57,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-26T19:13:57,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/B/725f293dbdb644f6b1a8146d0fcd5831 is 50, key is test_row_0/B:col10/1732648436782/Put/seqid=0 2024-11-26T19:13:57,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741877_1053 (size=12151) 2024-11-26T19:13:57,463 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:13:57,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49210 deadline: 1732648497462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:13:57,464 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:13:57,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49154 deadline: 1732648497462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:13:57,464 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to fe76acf732b2,42785,1732648423970 for 'test_row_2' in 76c4ff0c848e5ffb5ad9989537063596 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4211 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T19:13:57,465 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:57,465 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:57,465 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:13:57,465 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to fe76acf732b2,42785,1732648423970 for 'test_row_0' in 76c4ff0c848e5ffb5ad9989537063596 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4211 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T19:13:57,465 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:57,466 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:57,466 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:13:57,494 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:13:57,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49132 deadline: 1732648497492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:13:57,495 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:57,495 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:57,495 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:13:57,503 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:13:57,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49198 deadline: 1732648497502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:13:57,504 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:57,504 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:57,504 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:13:57,505 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:13:57,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49182 deadline: 1732648497503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:13:57,506 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:57,506 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:57,506 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:13:57,804 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:13:57,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49132 deadline: 1732648497802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:13:57,805 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:57,805 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:57,805 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:13:57,815 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:13:57,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49198 deadline: 1732648497812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:13:57,816 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:57,816 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:57,816 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:13:57,816 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:13:57,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49182 deadline: 1732648497812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:13:57,817 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/B/725f293dbdb644f6b1a8146d0fcd5831 2024-11-26T19:13:57,817 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:57,817 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:57,817 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:13:57,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/C/4c4d3d19acd0421e9abfd9d3b9021ab0 is 50, key is test_row_0/C:col10/1732648436782/Put/seqid=0 2024-11-26T19:13:57,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741878_1054 (size=12151) 2024-11-26T19:13:57,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-26T19:13:58,237 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/C/4c4d3d19acd0421e9abfd9d3b9021ab0 2024-11-26T19:13:58,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/A/6c9a6be2cdd34d3e80c6d012d0bddb33 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/6c9a6be2cdd34d3e80c6d012d0bddb33 2024-11-26T19:13:58,253 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/6c9a6be2cdd34d3e80c6d012d0bddb33, entries=150, sequenceid=212, filesize=11.9 K 2024-11-26T19:13:58,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/B/725f293dbdb644f6b1a8146d0fcd5831 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/725f293dbdb644f6b1a8146d0fcd5831 2024-11-26T19:13:58,261 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/725f293dbdb644f6b1a8146d0fcd5831, entries=150, sequenceid=212, filesize=11.9 K 2024-11-26T19:13:58,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-11-26T19:13:58,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/C/4c4d3d19acd0421e9abfd9d3b9021ab0 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/4c4d3d19acd0421e9abfd9d3b9021ab0 2024-11-26T19:13:58,271 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/4c4d3d19acd0421e9abfd9d3b9021ab0, entries=150, sequenceid=212, filesize=11.9 K 2024-11-26T19:13:58,272 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=107.34 KB/109920 for 76c4ff0c848e5ffb5ad9989537063596 in 1335ms, sequenceid=212, compaction requested=false 2024-11-26T19:13:58,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for 76c4ff0c848e5ffb5ad9989537063596: 2024-11-26T19:13:58,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. 2024-11-26T19:13:58,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-26T19:13:58,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-26T19:13:58,278 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-26T19:13:58,278 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4940 sec 2024-11-26T19:13:58,280 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 1.5050 sec 2024-11-26T19:13:58,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(8855): Flush requested on 76c4ff0c848e5ffb5ad9989537063596 2024-11-26T19:13:58,317 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 76c4ff0c848e5ffb5ad9989537063596 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-26T19:13:58,317 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c4ff0c848e5ffb5ad9989537063596, store=A 2024-11-26T19:13:58,317 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:13:58,318 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c4ff0c848e5ffb5ad9989537063596, store=B 2024-11-26T19:13:58,318 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:13:58,318 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c4ff0c848e5ffb5ad9989537063596, store=C 2024-11-26T19:13:58,318 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:13:58,324 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/A/71f2b009b2244b68ad864e49663a98cc is 50, key is test_row_0/A:col10/1732648437142/Put/seqid=0 2024-11-26T19:13:58,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741879_1055 (size=12151) 2024-11-26T19:13:58,343 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/A/71f2b009b2244b68ad864e49663a98cc 2024-11-26T19:13:58,354 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:13:58,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49182 deadline: 1732648498345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:13:58,355 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:58,355 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:58,355 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:13:58,358 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/B/13106d68607942e687f8bb2f54fa1a93 is 50, key is test_row_0/B:col10/1732648437142/Put/seqid=0 2024-11-26T19:13:58,361 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:13:58,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49132 deadline: 1732648498354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:13:58,362 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:58,362 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:13:58,362 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:58,362 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:13:58,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49198 deadline: 1732648498354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:13:58,363 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:58,363 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:58,363 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:13:58,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741880_1056 (size=12151) 2024-11-26T19:13:58,396 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/B/13106d68607942e687f8bb2f54fa1a93 2024-11-26T19:13:58,412 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/C/c4a004465289467ea564f3accff09f19 is 50, key is test_row_0/C:col10/1732648437142/Put/seqid=0 2024-11-26T19:13:58,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741881_1057 (size=12151) 2024-11-26T19:13:58,463 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:13:58,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49182 deadline: 1732648498462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:13:58,464 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:58,464 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:58,465 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:13:58,475 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:13:58,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49132 deadline: 1732648498472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:13:58,476 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:58,476 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:13:58,476 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:58,476 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:13:58,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49198 deadline: 1732648498472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:13:58,476 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:58,476 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:58,476 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:13:58,674 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:13:58,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49182 deadline: 1732648498672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:13:58,675 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:58,675 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:58,675 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:13:58,688 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:13:58,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49132 deadline: 1732648498682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:13:58,689 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:58,689 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:13:58,689 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:58,689 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:13:58,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49198 deadline: 1732648498683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:13:58,689 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:58,689 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:58,689 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:13:58,823 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/C/c4a004465289467ea564f3accff09f19 2024-11-26T19:13:58,830 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/A/71f2b009b2244b68ad864e49663a98cc as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/71f2b009b2244b68ad864e49663a98cc 2024-11-26T19:13:58,838 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/71f2b009b2244b68ad864e49663a98cc, entries=150, sequenceid=235, filesize=11.9 K 2024-11-26T19:13:58,839 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/B/13106d68607942e687f8bb2f54fa1a93 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/13106d68607942e687f8bb2f54fa1a93 2024-11-26T19:13:58,846 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/13106d68607942e687f8bb2f54fa1a93, entries=150, sequenceid=235, filesize=11.9 K 2024-11-26T19:13:58,848 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/C/c4a004465289467ea564f3accff09f19 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/c4a004465289467ea564f3accff09f19 2024-11-26T19:13:58,857 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/c4a004465289467ea564f3accff09f19, entries=150, sequenceid=235, filesize=11.9 K 2024-11-26T19:13:58,858 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for 76c4ff0c848e5ffb5ad9989537063596 in 541ms, sequenceid=235, compaction requested=true 2024-11-26T19:13:58,859 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 76c4ff0c848e5ffb5ad9989537063596: 2024-11-26T19:13:58,859 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T19:13:58,859 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 76c4ff0c848e5ffb5ad9989537063596:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T19:13:58,859 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:13:58,859 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T19:13:58,860 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 76c4ff0c848e5ffb5ad9989537063596:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T19:13:58,860 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:13:58,860 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 76c4ff0c848e5ffb5ad9989537063596:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T19:13:58,860 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T19:13:58,860 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T19:13:58,860 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1541): 76c4ff0c848e5ffb5ad9989537063596/A is initiating minor compaction (all files) 2024-11-26T19:13:58,861 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 76c4ff0c848e5ffb5ad9989537063596/A in TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. 2024-11-26T19:13:58,861 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/139e2b3b8af743bfbecc48835a1357dc, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/6c9a6be2cdd34d3e80c6d012d0bddb33, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/71f2b009b2244b68ad864e49663a98cc] into tmpdir=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp, totalSize=36.0 K 2024-11-26T19:13:58,861 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T19:13:58,861 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1541): 76c4ff0c848e5ffb5ad9989537063596/B is initiating minor compaction (all files) 2024-11-26T19:13:58,861 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 76c4ff0c848e5ffb5ad9989537063596/B in TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. 2024-11-26T19:13:58,861 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/7b79c14e7270430095e6008dcaeece65, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/725f293dbdb644f6b1a8146d0fcd5831, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/13106d68607942e687f8bb2f54fa1a93] into tmpdir=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp, totalSize=36.0 K 2024-11-26T19:13:58,862 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting 139e2b3b8af743bfbecc48835a1357dc, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1732648436099 2024-11-26T19:13:58,862 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting 7b79c14e7270430095e6008dcaeece65, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1732648436099 2024-11-26T19:13:58,862 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6c9a6be2cdd34d3e80c6d012d0bddb33, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1732648436759 2024-11-26T19:13:58,862 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting 725f293dbdb644f6b1a8146d0fcd5831, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1732648436759 2024-11-26T19:13:58,863 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting 71f2b009b2244b68ad864e49663a98cc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1732648437142 2024-11-26T19:13:58,863 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting 13106d68607942e687f8bb2f54fa1a93, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1732648437142 2024-11-26T19:13:58,875 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 76c4ff0c848e5ffb5ad9989537063596#A#compaction#45 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T19:13:58,876 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/A/23ad685edd2d4041813aec8a7b6602ef is 50, key is test_row_0/A:col10/1732648437142/Put/seqid=0 2024-11-26T19:13:58,877 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 76c4ff0c848e5ffb5ad9989537063596#B#compaction#46 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T19:13:58,877 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/B/f9e4dc2be254487292a3465fa91e842d is 50, key is test_row_0/B:col10/1732648437142/Put/seqid=0 2024-11-26T19:13:58,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741882_1058 (size=12663) 2024-11-26T19:13:58,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741883_1059 (size=12663) 2024-11-26T19:13:58,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-26T19:13:58,922 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T19:13:58,923 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T19:13:58,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-26T19:13:58,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-26T19:13:58,930 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T19:13:58,934 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T19:13:58,935 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T19:13:58,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(8855): Flush requested on 76c4ff0c848e5ffb5ad9989537063596 2024-11-26T19:13:58,985 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 76c4ff0c848e5ffb5ad9989537063596 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-26T19:13:58,985 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c4ff0c848e5ffb5ad9989537063596, store=A 2024-11-26T19:13:58,985 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:13:58,986 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c4ff0c848e5ffb5ad9989537063596, store=B 2024-11-26T19:13:58,986 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:13:58,986 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c4ff0c848e5ffb5ad9989537063596, store=C 2024-11-26T19:13:58,986 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:13:58,993 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/A/77d0a0ab017d44e9b68da403b4454e10 is 50, key is test_row_0/A:col10/1732648438338/Put/seqid=0 2024-11-26T19:13:59,011 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741884_1060 (size=14541) 2024-11-26T19:13:59,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-26T19:13:59,051 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:13:59,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49182 deadline: 1732648499041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:13:59,051 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:13:59,052 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:59,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49132 deadline: 1732648499043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:13:59,052 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:59,052 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:13:59,052 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:59,052 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:59,052 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:13:59,057 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:13:59,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49198 deadline: 1732648499051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:13:59,057 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:59,057 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:59,057 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:13:59,090 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-26T19:13:59,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. 2024-11-26T19:13:59,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. as already flushing 2024-11-26T19:13:59,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. 2024-11-26T19:13:59,091 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:13:59,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:13:59,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:13:59,164 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:13:59,164 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:13:59,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49132 deadline: 1732648499163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:13:59,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49182 deadline: 1732648499163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:13:59,165 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:13:59,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49198 deadline: 1732648499163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:13:59,165 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:59,165 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:59,165 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:59,165 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:59,165 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:13:59,165 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:13:59,166 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:59,166 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:59,166 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:13:59,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-26T19:13:59,245 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-26T19:13:59,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. 2024-11-26T19:13:59,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. as already flushing 2024-11-26T19:13:59,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. 2024-11-26T19:13:59,246 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:13:59,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:13:59,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:13:59,303 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/A/23ad685edd2d4041813aec8a7b6602ef as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/23ad685edd2d4041813aec8a7b6602ef 2024-11-26T19:13:59,303 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/B/f9e4dc2be254487292a3465fa91e842d as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/f9e4dc2be254487292a3465fa91e842d 2024-11-26T19:13:59,314 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 76c4ff0c848e5ffb5ad9989537063596/A of 76c4ff0c848e5ffb5ad9989537063596 into 23ad685edd2d4041813aec8a7b6602ef(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T19:13:59,314 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 76c4ff0c848e5ffb5ad9989537063596: 2024-11-26T19:13:59,314 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., storeName=76c4ff0c848e5ffb5ad9989537063596/A, priority=13, startTime=1732648438859; duration=0sec 2024-11-26T19:13:59,314 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T19:13:59,315 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 76c4ff0c848e5ffb5ad9989537063596:A 2024-11-26T19:13:59,315 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T19:13:59,316 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T19:13:59,316 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1541): 76c4ff0c848e5ffb5ad9989537063596/C is initiating minor compaction (all files) 2024-11-26T19:13:59,317 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 76c4ff0c848e5ffb5ad9989537063596/C in TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. 2024-11-26T19:13:59,317 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/ca33c2a4ec0749d19d651b83f18f98f5, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/4c4d3d19acd0421e9abfd9d3b9021ab0, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/c4a004465289467ea564f3accff09f19] into tmpdir=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp, totalSize=36.0 K 2024-11-26T19:13:59,317 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting ca33c2a4ec0749d19d651b83f18f98f5, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1732648436099 2024-11-26T19:13:59,318 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4c4d3d19acd0421e9abfd9d3b9021ab0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1732648436759 2024-11-26T19:13:59,318 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 76c4ff0c848e5ffb5ad9989537063596/B of 76c4ff0c848e5ffb5ad9989537063596 into f9e4dc2be254487292a3465fa91e842d(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T19:13:59,318 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 76c4ff0c848e5ffb5ad9989537063596: 2024-11-26T19:13:59,318 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., storeName=76c4ff0c848e5ffb5ad9989537063596/B, priority=13, startTime=1732648438859; duration=0sec 2024-11-26T19:13:59,319 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:13:59,319 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 76c4ff0c848e5ffb5ad9989537063596:B 2024-11-26T19:13:59,319 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting c4a004465289467ea564f3accff09f19, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1732648437142 2024-11-26T19:13:59,335 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 76c4ff0c848e5ffb5ad9989537063596#C#compaction#48 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T19:13:59,336 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/C/e128b829b48a4a2b965d6ac11efc84d9 is 50, key is test_row_0/C:col10/1732648437142/Put/seqid=0 2024-11-26T19:13:59,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741885_1061 (size=12663) 2024-11-26T19:13:59,375 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:13:59,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49182 deadline: 1732648499373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:13:59,376 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:13:59,377 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:59,377 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:59,377 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:13:59,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49198 deadline: 1732648499373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:13:59,377 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:13:59,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49132 deadline: 1732648499373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:13:59,378 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:59,378 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:59,378 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:13:59,380 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/C/e128b829b48a4a2b965d6ac11efc84d9 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/e128b829b48a4a2b965d6ac11efc84d9 2024-11-26T19:13:59,382 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:59,382 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:59,383 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:13:59,388 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 76c4ff0c848e5ffb5ad9989537063596/C of 76c4ff0c848e5ffb5ad9989537063596 into e128b829b48a4a2b965d6ac11efc84d9(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T19:13:59,388 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 76c4ff0c848e5ffb5ad9989537063596: 2024-11-26T19:13:59,388 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., storeName=76c4ff0c848e5ffb5ad9989537063596/C, priority=13, startTime=1732648438860; duration=0sec 2024-11-26T19:13:59,389 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:13:59,389 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 76c4ff0c848e5ffb5ad9989537063596:C 2024-11-26T19:13:59,401 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-26T19:13:59,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. 2024-11-26T19:13:59,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. as already flushing 2024-11-26T19:13:59,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. 2024-11-26T19:13:59,402 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:13:59,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:13:59,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:13:59,412 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/A/77d0a0ab017d44e9b68da403b4454e10 2024-11-26T19:13:59,426 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/B/59f264b3fff84b77822329d7c684aa42 is 50, key is test_row_0/B:col10/1732648438338/Put/seqid=0 2024-11-26T19:13:59,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741886_1062 (size=12151) 2024-11-26T19:13:59,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-26T19:13:59,555 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-26T19:13:59,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. 2024-11-26T19:13:59,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. as already flushing 2024-11-26T19:13:59,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. 2024-11-26T19:13:59,556 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:13:59,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:13:59,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:13:59,685 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:13:59,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49182 deadline: 1732648499681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:13:59,686 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:59,686 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:59,686 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:13:59,696 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:13:59,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49198 deadline: 1732648499692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:13:59,697 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:59,697 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:13:59,697 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:59,697 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:13:59,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49132 deadline: 1732648499692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:13:59,697 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:59,697 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:13:59,697 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:13:59,709 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-26T19:13:59,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. 2024-11-26T19:13:59,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. as already flushing 2024-11-26T19:13:59,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. 2024-11-26T19:13:59,710 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:13:59,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:13:59,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:13:59,832 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/B/59f264b3fff84b77822329d7c684aa42 2024-11-26T19:13:59,846 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/C/13d7bbfdd0b943cd9c869f98e44f7b4e is 50, key is test_row_0/C:col10/1732648438338/Put/seqid=0 2024-11-26T19:13:59,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741887_1063 (size=12151) 2024-11-26T19:13:59,864 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-26T19:13:59,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. 2024-11-26T19:13:59,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. as already flushing 2024-11-26T19:13:59,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. 2024-11-26T19:13:59,864 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:13:59,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:13:59,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:00,019 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-26T19:14:00,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. 2024-11-26T19:14:00,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. as already flushing 2024-11-26T19:14:00,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. 2024-11-26T19:14:00,019 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:00,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:00,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:00,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-26T19:14:00,173 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-26T19:14:00,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. 2024-11-26T19:14:00,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. as already flushing 2024-11-26T19:14:00,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. 2024-11-26T19:14:00,174 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:00,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:00,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:00,194 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:00,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49182 deadline: 1732648500192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:00,194 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:00,194 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:00,195 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:00,212 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:00,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49198 deadline: 1732648500212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:00,213 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:00,213 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:00,213 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:00,216 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:00,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49132 deadline: 1732648500212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:00,217 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:00,217 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:00,217 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:00,255 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/C/13d7bbfdd0b943cd9c869f98e44f7b4e 2024-11-26T19:14:00,262 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/A/77d0a0ab017d44e9b68da403b4454e10 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/77d0a0ab017d44e9b68da403b4454e10 2024-11-26T19:14:00,268 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/77d0a0ab017d44e9b68da403b4454e10, entries=200, sequenceid=249, filesize=14.2 K 2024-11-26T19:14:00,269 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/B/59f264b3fff84b77822329d7c684aa42 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/59f264b3fff84b77822329d7c684aa42 2024-11-26T19:14:00,277 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/59f264b3fff84b77822329d7c684aa42, entries=150, sequenceid=249, filesize=11.9 K 2024-11-26T19:14:00,278 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/C/13d7bbfdd0b943cd9c869f98e44f7b4e as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/13d7bbfdd0b943cd9c869f98e44f7b4e 2024-11-26T19:14:00,284 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/13d7bbfdd0b943cd9c869f98e44f7b4e, entries=150, sequenceid=249, filesize=11.9 K 2024-11-26T19:14:00,285 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 76c4ff0c848e5ffb5ad9989537063596 in 1300ms, sequenceid=249, compaction requested=false 2024-11-26T19:14:00,285 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 76c4ff0c848e5ffb5ad9989537063596: 2024-11-26T19:14:00,328 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-26T19:14:00,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. 2024-11-26T19:14:00,328 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing 76c4ff0c848e5ffb5ad9989537063596 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-26T19:14:00,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c4ff0c848e5ffb5ad9989537063596, store=A 2024-11-26T19:14:00,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:14:00,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c4ff0c848e5ffb5ad9989537063596, store=B 2024-11-26T19:14:00,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:14:00,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c4ff0c848e5ffb5ad9989537063596, store=C 2024-11-26T19:14:00,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:14:00,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/A/a97c2c7078f346d4866eb711c11b0796 is 50, key is test_row_0/A:col10/1732648439038/Put/seqid=0 2024-11-26T19:14:00,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741888_1064 (size=12301) 2024-11-26T19:14:00,743 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/A/a97c2c7078f346d4866eb711c11b0796 2024-11-26T19:14:00,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/B/8dfb3e1b0884467ba576198f7f349403 is 50, key is test_row_0/B:col10/1732648439038/Put/seqid=0 2024-11-26T19:14:00,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741889_1065 (size=12301) 2024-11-26T19:14:01,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-26T19:14:01,161 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/B/8dfb3e1b0884467ba576198f7f349403 2024-11-26T19:14:01,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/C/02c786736cee400e90920e425e21500b is 50, key is test_row_0/C:col10/1732648439038/Put/seqid=0 2024-11-26T19:14:01,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741890_1066 (size=12301) 2024-11-26T19:14:01,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(8855): Flush requested on 76c4ff0c848e5ffb5ad9989537063596 2024-11-26T19:14:01,206 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. as already flushing 2024-11-26T19:14:01,240 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:01,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49198 deadline: 1732648501235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:01,240 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:01,241 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:01,241 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:01,243 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:01,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49132 deadline: 1732648501238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:01,244 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:01,244 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:01,244 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:01,244 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:01,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49182 deadline: 1732648501240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:01,245 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:01,245 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:01,245 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:01,353 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:01,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49198 deadline: 1732648501352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:01,354 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:01,354 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:01,354 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:01,356 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:01,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49182 deadline: 1732648501353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:01,357 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:01,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49132 deadline: 1732648501353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:01,357 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:01,357 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:01,357 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:01,357 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:01,358 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:01,358 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:01,504 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:01,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49210 deadline: 1732648501502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:01,505 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to fe76acf732b2,42785,1732648423970 for 'test_row_2' in 76c4ff0c848e5ffb5ad9989537063596 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8252 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T19:14:01,506 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:01,506 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:01,506 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:01,512 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:01,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49154 deadline: 1732648501512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:01,513 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to fe76acf732b2,42785,1732648423970 for 'test_row_0' in 76c4ff0c848e5ffb5ad9989537063596 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8260 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T19:14:01,514 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:01,514 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:01,514 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:01,568 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:01,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49198 deadline: 1732648501561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:01,569 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:01,569 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:01,569 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:01,569 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:01,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49182 deadline: 1732648501562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:01,569 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:01,569 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:01,569 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:01,570 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:01,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49132 deadline: 1732648501562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:01,570 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:01,570 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:01,570 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:01,579 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/C/02c786736cee400e90920e425e21500b 2024-11-26T19:14:01,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/A/a97c2c7078f346d4866eb711c11b0796 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/a97c2c7078f346d4866eb711c11b0796 2024-11-26T19:14:01,593 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/a97c2c7078f346d4866eb711c11b0796, entries=150, sequenceid=270, filesize=12.0 K 2024-11-26T19:14:01,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/B/8dfb3e1b0884467ba576198f7f349403 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/8dfb3e1b0884467ba576198f7f349403 2024-11-26T19:14:01,601 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/8dfb3e1b0884467ba576198f7f349403, entries=150, sequenceid=270, filesize=12.0 K 2024-11-26T19:14:01,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/C/02c786736cee400e90920e425e21500b as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/02c786736cee400e90920e425e21500b 2024-11-26T19:14:01,611 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/02c786736cee400e90920e425e21500b, entries=150, sequenceid=270, filesize=12.0 K 2024-11-26T19:14:01,616 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 76c4ff0c848e5ffb5ad9989537063596 in 1288ms, sequenceid=270, compaction requested=true 2024-11-26T19:14:01,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for 76c4ff0c848e5ffb5ad9989537063596: 2024-11-26T19:14:01,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. 2024-11-26T19:14:01,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-26T19:14:01,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-26T19:14:01,623 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-26T19:14:01,623 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6850 sec 2024-11-26T19:14:01,626 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 2.7010 sec 2024-11-26T19:14:01,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(8855): Flush requested on 76c4ff0c848e5ffb5ad9989537063596 2024-11-26T19:14:01,878 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 76c4ff0c848e5ffb5ad9989537063596 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-26T19:14:01,880 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c4ff0c848e5ffb5ad9989537063596, store=A 2024-11-26T19:14:01,880 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:14:01,880 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c4ff0c848e5ffb5ad9989537063596, store=B 2024-11-26T19:14:01,880 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:14:01,880 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c4ff0c848e5ffb5ad9989537063596, store=C 2024-11-26T19:14:01,880 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:14:01,888 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/A/d24b66db5ff54903a97bf0b7d04b52a0 is 50, key is test_row_0/A:col10/1732648441238/Put/seqid=0 2024-11-26T19:14:01,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741891_1067 (size=14741) 2024-11-26T19:14:01,903 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/A/d24b66db5ff54903a97bf0b7d04b52a0 2024-11-26T19:14:01,916 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/B/d41bfc4dd3d348028897b897e60c3af3 is 50, key is test_row_0/B:col10/1732648441238/Put/seqid=0 2024-11-26T19:14:01,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741892_1068 (size=12301) 2024-11-26T19:14:01,926 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:01,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49132 deadline: 1732648501919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:01,927 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:01,927 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:01,927 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:01,927 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:01,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49182 deadline: 1732648501919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:01,927 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/B/d41bfc4dd3d348028897b897e60c3af3 2024-11-26T19:14:01,928 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:01,928 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:01,928 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:01,928 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:01,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49198 deadline: 1732648501922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:01,928 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:01,928 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:01,928 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:01,959 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/C/8d8b1a4596ab4d139bdbf13a5c9e37bc is 50, key is test_row_0/C:col10/1732648441238/Put/seqid=0 2024-11-26T19:14:01,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741893_1069 (size=12301) 2024-11-26T19:14:02,034 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:02,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49132 deadline: 1732648502032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:02,035 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:02,035 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:02,035 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:02,035 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:02,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49182 deadline: 1732648502032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:02,035 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:02,035 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:02,035 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:02,035 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:02,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49198 deadline: 1732648502032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:02,036 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:02,036 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:02,036 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:02,242 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:02,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49132 deadline: 1732648502242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:02,243 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:02,243 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:02,243 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:02,243 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:02,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49182 deadline: 1732648502242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:02,244 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:02,244 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:02,244 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:02,244 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:02,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49198 deadline: 1732648502242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:02,244 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:02,244 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:02,244 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:02,367 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/C/8d8b1a4596ab4d139bdbf13a5c9e37bc 2024-11-26T19:14:02,375 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/A/d24b66db5ff54903a97bf0b7d04b52a0 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/d24b66db5ff54903a97bf0b7d04b52a0 2024-11-26T19:14:02,381 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/d24b66db5ff54903a97bf0b7d04b52a0, entries=200, sequenceid=286, filesize=14.4 K 2024-11-26T19:14:02,382 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/B/d41bfc4dd3d348028897b897e60c3af3 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/d41bfc4dd3d348028897b897e60c3af3 2024-11-26T19:14:02,387 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/d41bfc4dd3d348028897b897e60c3af3, entries=150, sequenceid=286, filesize=12.0 K 2024-11-26T19:14:02,388 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/C/8d8b1a4596ab4d139bdbf13a5c9e37bc as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/8d8b1a4596ab4d139bdbf13a5c9e37bc 2024-11-26T19:14:02,398 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/8d8b1a4596ab4d139bdbf13a5c9e37bc, entries=150, sequenceid=286, filesize=12.0 K 2024-11-26T19:14:02,400 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 76c4ff0c848e5ffb5ad9989537063596 in 521ms, sequenceid=286, compaction requested=true 2024-11-26T19:14:02,400 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 76c4ff0c848e5ffb5ad9989537063596: 2024-11-26T19:14:02,400 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 76c4ff0c848e5ffb5ad9989537063596:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T19:14:02,400 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:14:02,400 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T19:14:02,400 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T19:14:02,401 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 76c4ff0c848e5ffb5ad9989537063596:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T19:14:02,401 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:14:02,401 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 76c4ff0c848e5ffb5ad9989537063596:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T19:14:02,401 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T19:14:02,403 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54246 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T19:14:02,403 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49416 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T19:14:02,403 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1541): 76c4ff0c848e5ffb5ad9989537063596/B is initiating minor compaction (all files) 2024-11-26T19:14:02,403 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 76c4ff0c848e5ffb5ad9989537063596/B in TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. 2024-11-26T19:14:02,403 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/f9e4dc2be254487292a3465fa91e842d, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/59f264b3fff84b77822329d7c684aa42, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/8dfb3e1b0884467ba576198f7f349403, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/d41bfc4dd3d348028897b897e60c3af3] into tmpdir=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp, totalSize=48.3 K 2024-11-26T19:14:02,407 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting f9e4dc2be254487292a3465fa91e842d, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1732648437142 2024-11-26T19:14:02,409 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting 59f264b3fff84b77822329d7c684aa42, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1732648438338 2024-11-26T19:14:02,409 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting 8dfb3e1b0884467ba576198f7f349403, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1732648439036 2024-11-26T19:14:02,409 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1541): 76c4ff0c848e5ffb5ad9989537063596/A is initiating minor compaction (all files) 2024-11-26T19:14:02,409 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 76c4ff0c848e5ffb5ad9989537063596/A in TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. 2024-11-26T19:14:02,410 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/23ad685edd2d4041813aec8a7b6602ef, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/77d0a0ab017d44e9b68da403b4454e10, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/a97c2c7078f346d4866eb711c11b0796, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/d24b66db5ff54903a97bf0b7d04b52a0] into tmpdir=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp, totalSize=53.0 K 2024-11-26T19:14:02,410 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting d41bfc4dd3d348028897b897e60c3af3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1732648441220 2024-11-26T19:14:02,410 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting 23ad685edd2d4041813aec8a7b6602ef, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1732648437142 2024-11-26T19:14:02,411 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting 77d0a0ab017d44e9b68da403b4454e10, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1732648438338 2024-11-26T19:14:02,412 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting a97c2c7078f346d4866eb711c11b0796, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1732648439036 2024-11-26T19:14:02,412 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting d24b66db5ff54903a97bf0b7d04b52a0, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1732648441220 2024-11-26T19:14:02,435 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 76c4ff0c848e5ffb5ad9989537063596#B#compaction#57 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T19:14:02,436 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/B/da15be7eff4449a1807670adea92ff11 is 50, key is test_row_0/B:col10/1732648441238/Put/seqid=0 2024-11-26T19:14:02,451 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 76c4ff0c848e5ffb5ad9989537063596#A#compaction#58 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T19:14:02,452 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/A/6b65df8e384447379e3b1e045e6846e5 is 50, key is test_row_0/A:col10/1732648441238/Put/seqid=0 2024-11-26T19:14:02,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741894_1070 (size=12949) 2024-11-26T19:14:02,470 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/B/da15be7eff4449a1807670adea92ff11 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/da15be7eff4449a1807670adea92ff11 2024-11-26T19:14:02,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741895_1071 (size=12949) 2024-11-26T19:14:02,482 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 76c4ff0c848e5ffb5ad9989537063596/B of 76c4ff0c848e5ffb5ad9989537063596 into da15be7eff4449a1807670adea92ff11(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T19:14:02,482 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 76c4ff0c848e5ffb5ad9989537063596: 2024-11-26T19:14:02,482 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., storeName=76c4ff0c848e5ffb5ad9989537063596/B, priority=12, startTime=1732648442400; duration=0sec 2024-11-26T19:14:02,483 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T19:14:02,483 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 76c4ff0c848e5ffb5ad9989537063596:B 2024-11-26T19:14:02,483 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T19:14:02,490 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/A/6b65df8e384447379e3b1e045e6846e5 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/6b65df8e384447379e3b1e045e6846e5 2024-11-26T19:14:02,500 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49416 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T19:14:02,500 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1541): 76c4ff0c848e5ffb5ad9989537063596/C is initiating minor compaction (all files) 2024-11-26T19:14:02,500 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 76c4ff0c848e5ffb5ad9989537063596/C in TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. 2024-11-26T19:14:02,501 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/e128b829b48a4a2b965d6ac11efc84d9, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/13d7bbfdd0b943cd9c869f98e44f7b4e, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/02c786736cee400e90920e425e21500b, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/8d8b1a4596ab4d139bdbf13a5c9e37bc] into tmpdir=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp, totalSize=48.3 K 2024-11-26T19:14:02,501 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting e128b829b48a4a2b965d6ac11efc84d9, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1732648437142 2024-11-26T19:14:02,501 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 76c4ff0c848e5ffb5ad9989537063596/A of 76c4ff0c848e5ffb5ad9989537063596 into 6b65df8e384447379e3b1e045e6846e5(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T19:14:02,501 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 76c4ff0c848e5ffb5ad9989537063596: 2024-11-26T19:14:02,501 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., storeName=76c4ff0c848e5ffb5ad9989537063596/A, priority=12, startTime=1732648442400; duration=0sec 2024-11-26T19:14:02,502 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting 13d7bbfdd0b943cd9c869f98e44f7b4e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1732648438338 2024-11-26T19:14:02,502 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:14:02,502 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 76c4ff0c848e5ffb5ad9989537063596:A 2024-11-26T19:14:02,505 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting 02c786736cee400e90920e425e21500b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1732648439036 2024-11-26T19:14:02,505 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting 8d8b1a4596ab4d139bdbf13a5c9e37bc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1732648441220 2024-11-26T19:14:02,523 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 76c4ff0c848e5ffb5ad9989537063596#C#compaction#59 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T19:14:02,524 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/C/fbb60340b5c64f80b794ae598ca80a8f is 50, key is test_row_0/C:col10/1732648441238/Put/seqid=0 2024-11-26T19:14:02,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741896_1072 (size=12949) 2024-11-26T19:14:02,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(8855): Flush requested on 76c4ff0c848e5ffb5ad9989537063596 2024-11-26T19:14:02,557 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 76c4ff0c848e5ffb5ad9989537063596 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-26T19:14:02,558 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c4ff0c848e5ffb5ad9989537063596, store=A 2024-11-26T19:14:02,558 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:14:02,558 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c4ff0c848e5ffb5ad9989537063596, store=B 2024-11-26T19:14:02,558 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:14:02,558 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c4ff0c848e5ffb5ad9989537063596, store=C 2024-11-26T19:14:02,558 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:14:02,564 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/A/9cc0e8b7ca2341f9a661a7c6af090fa6 is 50, key is test_row_0/A:col10/1732648441915/Put/seqid=0 2024-11-26T19:14:02,590 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:02,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49198 deadline: 1732648502582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:02,591 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:02,591 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:02,591 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:02,591 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:02,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49182 deadline: 1732648502583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:02,591 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:02,591 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:02,591 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:02,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741897_1073 (size=12301) 2024-11-26T19:14:02,594 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:02,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49132 deadline: 1732648502590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:02,595 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:02,595 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:02,595 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:02,703 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:02,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49132 deadline: 1732648502702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:02,703 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:02,703 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:02,703 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:02,704 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:02,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49182 deadline: 1732648502702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:02,704 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:02,704 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:02,704 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:02,704 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:02,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49198 deadline: 1732648502702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:02,705 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:02,705 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:02,705 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:02,914 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:02,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49132 deadline: 1732648502912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:02,915 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:02,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49182 deadline: 1732648502913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:02,916 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:02,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49198 deadline: 1732648502913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:02,916 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:02,916 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:02,916 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:02,916 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:02,916 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:02,916 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:02,917 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:02,917 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:02,917 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:02,956 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/C/fbb60340b5c64f80b794ae598ca80a8f as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/fbb60340b5c64f80b794ae598ca80a8f 2024-11-26T19:14:02,967 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 76c4ff0c848e5ffb5ad9989537063596/C of 76c4ff0c848e5ffb5ad9989537063596 into fbb60340b5c64f80b794ae598ca80a8f(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T19:14:02,967 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 76c4ff0c848e5ffb5ad9989537063596: 2024-11-26T19:14:02,967 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., storeName=76c4ff0c848e5ffb5ad9989537063596/C, priority=12, startTime=1732648442401; duration=0sec 2024-11-26T19:14:02,967 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:14:02,967 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 76c4ff0c848e5ffb5ad9989537063596:C 2024-11-26T19:14:02,994 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=306 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/A/9cc0e8b7ca2341f9a661a7c6af090fa6 2024-11-26T19:14:03,010 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/B/ff81f2d8e01346e1b7b68b1892e6f8fb is 50, key is test_row_0/B:col10/1732648441915/Put/seqid=0 2024-11-26T19:14:03,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741898_1074 (size=12301) 2024-11-26T19:14:03,021 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=306 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/B/ff81f2d8e01346e1b7b68b1892e6f8fb 2024-11-26T19:14:03,039 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/C/a6683e1a65124d1bb009ce53628acb6a is 50, key is test_row_0/C:col10/1732648441915/Put/seqid=0 2024-11-26T19:14:03,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-26T19:14:03,083 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T19:14:03,083 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T19:14:03,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-26T19:14:03,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-26T19:14:03,086 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T19:14:03,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741899_1075 (size=12301) 2024-11-26T19:14:03,088 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T19:14:03,088 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T19:14:03,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-26T19:14:03,225 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:03,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49182 deadline: 1732648503222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:03,226 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:03,226 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:03,226 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:03,226 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:03,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49198 deadline: 1732648503222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:03,226 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:03,227 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:03,227 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:03,227 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:03,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49132 deadline: 1732648503222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:03,227 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:03,227 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:03,227 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:03,243 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-26T19:14:03,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. 2024-11-26T19:14:03,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. as already flushing 2024-11-26T19:14:03,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. 2024-11-26T19:14:03,244 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:03,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:03,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:03,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-26T19:14:03,402 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-26T19:14:03,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. 2024-11-26T19:14:03,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. as already flushing 2024-11-26T19:14:03,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. 2024-11-26T19:14:03,402 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:03,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:03,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:03,487 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=306 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/C/a6683e1a65124d1bb009ce53628acb6a 2024-11-26T19:14:03,501 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/A/9cc0e8b7ca2341f9a661a7c6af090fa6 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/9cc0e8b7ca2341f9a661a7c6af090fa6 2024-11-26T19:14:03,509 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/9cc0e8b7ca2341f9a661a7c6af090fa6, entries=150, sequenceid=306, filesize=12.0 K 2024-11-26T19:14:03,511 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/B/ff81f2d8e01346e1b7b68b1892e6f8fb as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/ff81f2d8e01346e1b7b68b1892e6f8fb 2024-11-26T19:14:03,519 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/ff81f2d8e01346e1b7b68b1892e6f8fb, entries=150, sequenceid=306, filesize=12.0 K 2024-11-26T19:14:03,520 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/C/a6683e1a65124d1bb009ce53628acb6a as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/a6683e1a65124d1bb009ce53628acb6a 2024-11-26T19:14:03,533 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/a6683e1a65124d1bb009ce53628acb6a, entries=150, sequenceid=306, filesize=12.0 K 2024-11-26T19:14:03,534 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 76c4ff0c848e5ffb5ad9989537063596 in 977ms, sequenceid=306, compaction requested=false 2024-11-26T19:14:03,534 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 76c4ff0c848e5ffb5ad9989537063596: 2024-11-26T19:14:03,556 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-26T19:14:03,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. 2024-11-26T19:14:03,557 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing 76c4ff0c848e5ffb5ad9989537063596 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-26T19:14:03,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c4ff0c848e5ffb5ad9989537063596, store=A 2024-11-26T19:14:03,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:14:03,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c4ff0c848e5ffb5ad9989537063596, store=B 2024-11-26T19:14:03,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:14:03,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c4ff0c848e5ffb5ad9989537063596, store=C 2024-11-26T19:14:03,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:14:03,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/A/5b1c7d96ac094fe8a988e5dc7ea8f379 is 50, key is test_row_0/A:col10/1732648442588/Put/seqid=0 2024-11-26T19:14:03,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741900_1076 (size=12301) 2024-11-26T19:14:03,584 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=321 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/A/5b1c7d96ac094fe8a988e5dc7ea8f379 2024-11-26T19:14:03,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/B/73626e1537194c10bfe236f3d7188198 is 50, key is test_row_0/B:col10/1732648442588/Put/seqid=0 2024-11-26T19:14:03,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741901_1077 (size=12301) 2024-11-26T19:14:03,628 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=321 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/B/73626e1537194c10bfe236f3d7188198 2024-11-26T19:14:03,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/C/3faf0f0527fa4971a495694e2ba1081a is 50, key is test_row_0/C:col10/1732648442588/Put/seqid=0 2024-11-26T19:14:03,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741902_1078 (size=12301) 2024-11-26T19:14:03,658 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=321 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/C/3faf0f0527fa4971a495694e2ba1081a 2024-11-26T19:14:03,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/A/5b1c7d96ac094fe8a988e5dc7ea8f379 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/5b1c7d96ac094fe8a988e5dc7ea8f379 2024-11-26T19:14:03,672 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/5b1c7d96ac094fe8a988e5dc7ea8f379, entries=150, sequenceid=321, filesize=12.0 K 2024-11-26T19:14:03,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/B/73626e1537194c10bfe236f3d7188198 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/73626e1537194c10bfe236f3d7188198 2024-11-26T19:14:03,681 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/73626e1537194c10bfe236f3d7188198, entries=150, sequenceid=321, filesize=12.0 K 2024-11-26T19:14:03,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/C/3faf0f0527fa4971a495694e2ba1081a as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/3faf0f0527fa4971a495694e2ba1081a 2024-11-26T19:14:03,689 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/3faf0f0527fa4971a495694e2ba1081a, entries=150, sequenceid=321, filesize=12.0 K 2024-11-26T19:14:03,691 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=0 B/0 for 76c4ff0c848e5ffb5ad9989537063596 in 134ms, sequenceid=321, compaction requested=true 2024-11-26T19:14:03,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for 76c4ff0c848e5ffb5ad9989537063596: 2024-11-26T19:14:03,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. 2024-11-26T19:14:03,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-26T19:14:03,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-26T19:14:03,697 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-26T19:14:03,697 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 605 msec 2024-11-26T19:14:03,701 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 616 msec 2024-11-26T19:14:03,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-26T19:14:03,712 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T19:14:03,713 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T19:14:03,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-26T19:14:03,717 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T19:14:03,718 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T19:14:03,718 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T19:14:03,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-26T19:14:03,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(8855): Flush requested on 76c4ff0c848e5ffb5ad9989537063596 2024-11-26T19:14:03,749 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 76c4ff0c848e5ffb5ad9989537063596 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-26T19:14:03,749 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c4ff0c848e5ffb5ad9989537063596, store=A 2024-11-26T19:14:03,749 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:14:03,749 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c4ff0c848e5ffb5ad9989537063596, store=B 2024-11-26T19:14:03,750 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:14:03,750 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c4ff0c848e5ffb5ad9989537063596, store=C 2024-11-26T19:14:03,750 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:14:03,757 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/A/421ad5e1c9c84d8684b7ef8e6d6d76f8 is 50, key is test_row_0/A:col10/1732648443732/Put/seqid=0 2024-11-26T19:14:03,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741903_1079 (size=17181) 2024-11-26T19:14:03,771 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=331 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/A/421ad5e1c9c84d8684b7ef8e6d6d76f8 2024-11-26T19:14:03,789 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/B/32f6f58cb60847a5983a76e858f37cdd is 50, key is test_row_0/B:col10/1732648443732/Put/seqid=0 2024-11-26T19:14:03,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741904_1080 (size=12301) 2024-11-26T19:14:03,798 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=331 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/B/32f6f58cb60847a5983a76e858f37cdd 2024-11-26T19:14:03,818 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732648443792, max timestamp=1732648443813Num uniques 859; 2024-11-26T19:14:03,818 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732648443792, max timestamp=1732648443813Num uniques 859; 2024-11-26T19:14:03,818 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-26T19:14:03,818 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732648443792, max timestamp=1732648443813Num uniques 859; 2024-11-26T19:14:03,818 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-26T19:14:03,818 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-26T19:14:03,823 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/C/f6b18b9166df4406a886ed525bf517d5 is 50, key is test_row_0/C:col10/1732648443732/Put/seqid=0 2024-11-26T19:14:03,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-26T19:14:03,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741905_1081 (size=12301) 2024-11-26T19:14:03,847 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=331 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/C/f6b18b9166df4406a886ed525bf517d5 2024-11-26T19:14:03,855 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/A/421ad5e1c9c84d8684b7ef8e6d6d76f8 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/421ad5e1c9c84d8684b7ef8e6d6d76f8 2024-11-26T19:14:03,862 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/421ad5e1c9c84d8684b7ef8e6d6d76f8, entries=250, sequenceid=331, filesize=16.8 K 2024-11-26T19:14:03,864 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/B/32f6f58cb60847a5983a76e858f37cdd as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/32f6f58cb60847a5983a76e858f37cdd 2024-11-26T19:14:03,871 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:03,871 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:03,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49198 deadline: 1732648503862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:03,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49132 deadline: 1732648503862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:03,872 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:03,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49182 deadline: 1732648503864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:03,872 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:03,872 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:03,872 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:03,872 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:03,873 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:03,873 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:03,873 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-26T19:14:03,873 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:03,873 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:03,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. 2024-11-26T19:14:03,873 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:03,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. as already flushing 2024-11-26T19:14:03,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. 2024-11-26T19:14:03,873 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:03,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:03,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:03,875 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/32f6f58cb60847a5983a76e858f37cdd, entries=150, sequenceid=331, filesize=12.0 K 2024-11-26T19:14:03,877 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/C/f6b18b9166df4406a886ed525bf517d5 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/f6b18b9166df4406a886ed525bf517d5 2024-11-26T19:14:03,882 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/f6b18b9166df4406a886ed525bf517d5, entries=150, sequenceid=331, filesize=12.0 K 2024-11-26T19:14:03,883 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 76c4ff0c848e5ffb5ad9989537063596 in 134ms, sequenceid=331, compaction requested=true 2024-11-26T19:14:03,883 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 76c4ff0c848e5ffb5ad9989537063596: 2024-11-26T19:14:03,883 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 76c4ff0c848e5ffb5ad9989537063596:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T19:14:03,883 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:14:03,883 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T19:14:03,883 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 76c4ff0c848e5ffb5ad9989537063596:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T19:14:03,883 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:14:03,883 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 76c4ff0c848e5ffb5ad9989537063596:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T19:14:03,883 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T19:14:03,883 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T19:14:03,885 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54732 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T19:14:03,885 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1541): 76c4ff0c848e5ffb5ad9989537063596/A is initiating minor compaction (all files) 2024-11-26T19:14:03,885 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49852 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T19:14:03,885 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 76c4ff0c848e5ffb5ad9989537063596/A in TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. 2024-11-26T19:14:03,885 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1541): 76c4ff0c848e5ffb5ad9989537063596/B is initiating minor compaction (all files) 2024-11-26T19:14:03,885 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/6b65df8e384447379e3b1e045e6846e5, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/9cc0e8b7ca2341f9a661a7c6af090fa6, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/5b1c7d96ac094fe8a988e5dc7ea8f379, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/421ad5e1c9c84d8684b7ef8e6d6d76f8] into tmpdir=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp, totalSize=53.4 K 2024-11-26T19:14:03,885 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 76c4ff0c848e5ffb5ad9989537063596/B in TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. 2024-11-26T19:14:03,885 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/da15be7eff4449a1807670adea92ff11, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/ff81f2d8e01346e1b7b68b1892e6f8fb, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/73626e1537194c10bfe236f3d7188198, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/32f6f58cb60847a5983a76e858f37cdd] into tmpdir=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp, totalSize=48.7 K 2024-11-26T19:14:03,886 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6b65df8e384447379e3b1e045e6846e5, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1732648441220 2024-11-26T19:14:03,886 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting da15be7eff4449a1807670adea92ff11, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1732648441220 2024-11-26T19:14:03,886 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9cc0e8b7ca2341f9a661a7c6af090fa6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1732648441915 2024-11-26T19:14:03,886 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting ff81f2d8e01346e1b7b68b1892e6f8fb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1732648441915 2024-11-26T19:14:03,886 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5b1c7d96ac094fe8a988e5dc7ea8f379, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1732648442572 2024-11-26T19:14:03,887 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting 421ad5e1c9c84d8684b7ef8e6d6d76f8, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1732648443732 2024-11-26T19:14:03,888 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting 73626e1537194c10bfe236f3d7188198, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1732648442572 2024-11-26T19:14:03,889 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting 32f6f58cb60847a5983a76e858f37cdd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1732648443732 2024-11-26T19:14:03,900 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 76c4ff0c848e5ffb5ad9989537063596#A#compaction#69 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T19:14:03,900 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/A/2e4e6f7645ad44c583b88cf3d2644c69 is 50, key is test_row_0/A:col10/1732648443732/Put/seqid=0 2024-11-26T19:14:03,910 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 76c4ff0c848e5ffb5ad9989537063596#B#compaction#70 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T19:14:03,911 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/B/478b3346f9f842da835187dcdf211b46 is 50, key is test_row_0/B:col10/1732648443732/Put/seqid=0 2024-11-26T19:14:03,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741906_1082 (size=13085) 2024-11-26T19:14:03,923 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/A/2e4e6f7645ad44c583b88cf3d2644c69 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/2e4e6f7645ad44c583b88cf3d2644c69 2024-11-26T19:14:03,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741907_1083 (size=13085) 2024-11-26T19:14:03,933 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 76c4ff0c848e5ffb5ad9989537063596/A of 76c4ff0c848e5ffb5ad9989537063596 into 2e4e6f7645ad44c583b88cf3d2644c69(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T19:14:03,933 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 76c4ff0c848e5ffb5ad9989537063596: 2024-11-26T19:14:03,933 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/B/478b3346f9f842da835187dcdf211b46 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/478b3346f9f842da835187dcdf211b46 2024-11-26T19:14:03,933 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., storeName=76c4ff0c848e5ffb5ad9989537063596/A, priority=12, startTime=1732648443883; duration=0sec 2024-11-26T19:14:03,934 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T19:14:03,934 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 76c4ff0c848e5ffb5ad9989537063596:A 2024-11-26T19:14:03,934 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T19:14:03,938 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49852 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T19:14:03,938 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1541): 76c4ff0c848e5ffb5ad9989537063596/C is initiating minor compaction (all files) 2024-11-26T19:14:03,938 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 76c4ff0c848e5ffb5ad9989537063596/C in TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. 2024-11-26T19:14:03,938 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/fbb60340b5c64f80b794ae598ca80a8f, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/a6683e1a65124d1bb009ce53628acb6a, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/3faf0f0527fa4971a495694e2ba1081a, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/f6b18b9166df4406a886ed525bf517d5] into tmpdir=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp, totalSize=48.7 K 2024-11-26T19:14:03,939 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting fbb60340b5c64f80b794ae598ca80a8f, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1732648441220 2024-11-26T19:14:03,939 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting a6683e1a65124d1bb009ce53628acb6a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1732648441915 2024-11-26T19:14:03,939 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3faf0f0527fa4971a495694e2ba1081a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1732648442572 2024-11-26T19:14:03,940 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting f6b18b9166df4406a886ed525bf517d5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1732648443732 2024-11-26T19:14:03,943 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 76c4ff0c848e5ffb5ad9989537063596/B of 76c4ff0c848e5ffb5ad9989537063596 into 478b3346f9f842da835187dcdf211b46(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T19:14:03,943 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 76c4ff0c848e5ffb5ad9989537063596: 2024-11-26T19:14:03,943 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., storeName=76c4ff0c848e5ffb5ad9989537063596/B, priority=12, startTime=1732648443883; duration=0sec 2024-11-26T19:14:03,943 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:14:03,944 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 76c4ff0c848e5ffb5ad9989537063596:B 2024-11-26T19:14:03,954 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 76c4ff0c848e5ffb5ad9989537063596#C#compaction#71 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T19:14:03,955 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/C/6fab4748f5394c0eb9109fb7f84ecbaf is 50, key is test_row_0/C:col10/1732648443732/Put/seqid=0 2024-11-26T19:14:03,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741908_1084 (size=13085) 2024-11-26T19:14:03,984 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/C/6fab4748f5394c0eb9109fb7f84ecbaf as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/6fab4748f5394c0eb9109fb7f84ecbaf 2024-11-26T19:14:03,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(8855): Flush requested on 76c4ff0c848e5ffb5ad9989537063596 2024-11-26T19:14:03,990 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 76c4ff0c848e5ffb5ad9989537063596 3/3 column families, dataSize=140.89 KB heapSize=395.53 KB 2024-11-26T19:14:03,991 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c4ff0c848e5ffb5ad9989537063596, store=A 2024-11-26T19:14:03,991 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T19:14:03,991 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c4ff0c848e5ffb5ad9989537063596, store=B 2024-11-26T19:14:03,991 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T19:14:03,991 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c4ff0c848e5ffb5ad9989537063596, store=C 2024-11-26T19:14:03,991 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T19:14:03,993 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 76c4ff0c848e5ffb5ad9989537063596/C of 76c4ff0c848e5ffb5ad9989537063596 into 6fab4748f5394c0eb9109fb7f84ecbaf(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T19:14:03,993 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 76c4ff0c848e5ffb5ad9989537063596: 2024-11-26T19:14:03,993 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., storeName=76c4ff0c848e5ffb5ad9989537063596/C, priority=12, startTime=1732648443883; duration=0sec 2024-11-26T19:14:03,993 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:14:03,993 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 76c4ff0c848e5ffb5ad9989537063596:C 2024-11-26T19:14:04,001 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/A/58dc31274336443fb6448a3bb609ce64 is 50, key is test_row_0/A:col10/1732648443988/Put/seqid=0 2024-11-26T19:14:04,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741909_1085 (size=14741) 2024-11-26T19:14:04,016 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=369 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/A/58dc31274336443fb6448a3bb609ce64 2024-11-26T19:14:04,015 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:04,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49198 deadline: 1732648504004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:04,017 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:04,017 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:04,017 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:04,017 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:04,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49182 deadline: 1732648504015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:04,018 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:04,018 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:04,018 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:04,018 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:04,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49132 deadline: 1732648504017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:04,019 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:04,019 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:04,019 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:04,027 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-26T19:14:04,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. 2024-11-26T19:14:04,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. as already flushing 2024-11-26T19:14:04,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. 2024-11-26T19:14:04,027 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:04,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:04,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:04,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-26T19:14:04,036 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/B/8bfed9f8d1dd41f392250f028c36e710 is 50, key is test_row_0/B:col10/1732648443988/Put/seqid=0 2024-11-26T19:14:04,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741910_1086 (size=12301) 2024-11-26T19:14:04,051 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=369 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/B/8bfed9f8d1dd41f392250f028c36e710 2024-11-26T19:14:04,066 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/C/9adcb6f4199242698dfb6b3e69bdac63 is 50, key is test_row_0/C:col10/1732648443988/Put/seqid=0 2024-11-26T19:14:04,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741911_1087 (size=12301) 2024-11-26T19:14:04,122 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:04,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49182 deadline: 1732648504122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:04,123 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:04,123 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:04,123 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:04,123 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:04,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49198 deadline: 1732648504122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:04,124 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:04,124 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:04,124 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:04,125 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:04,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49132 deadline: 1732648504122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:04,127 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:04,127 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:04,127 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:04,181 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-26T19:14:04,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. 2024-11-26T19:14:04,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. as already flushing 2024-11-26T19:14:04,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. 2024-11-26T19:14:04,181 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:04,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:04,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:04,336 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:04,336 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-26T19:14:04,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49182 deadline: 1732648504332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:04,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. 2024-11-26T19:14:04,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. as already flushing 2024-11-26T19:14:04,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. 2024-11-26T19:14:04,337 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:04,337 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:04,337 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:04,337 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:04,337 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:04,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49198 deadline: 1732648504332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:04,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:04,337 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:04,337 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:04,337 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:04,337 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:04,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49132 deadline: 1732648504332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:04,338 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:04,338 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:04,338 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:04,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:04,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-26T19:14:04,481 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=369 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/C/9adcb6f4199242698dfb6b3e69bdac63 2024-11-26T19:14:04,487 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/A/58dc31274336443fb6448a3bb609ce64 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/58dc31274336443fb6448a3bb609ce64 2024-11-26T19:14:04,491 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-26T19:14:04,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. 2024-11-26T19:14:04,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. as already flushing 2024-11-26T19:14:04,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. 2024-11-26T19:14:04,492 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:04,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:04,493 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/58dc31274336443fb6448a3bb609ce64, entries=200, sequenceid=369, filesize=14.4 K 2024-11-26T19:14:04,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:04,494 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/B/8bfed9f8d1dd41f392250f028c36e710 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/8bfed9f8d1dd41f392250f028c36e710 2024-11-26T19:14:04,500 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/8bfed9f8d1dd41f392250f028c36e710, entries=150, sequenceid=369, filesize=12.0 K 2024-11-26T19:14:04,501 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/C/9adcb6f4199242698dfb6b3e69bdac63 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/9adcb6f4199242698dfb6b3e69bdac63 2024-11-26T19:14:04,510 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/9adcb6f4199242698dfb6b3e69bdac63, entries=150, sequenceid=369, filesize=12.0 K 2024-11-26T19:14:04,512 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~395.48 KB/404976, currentSize=46.96 KB/48090 for 76c4ff0c848e5ffb5ad9989537063596 in 521ms, sequenceid=369, compaction requested=false 2024-11-26T19:14:04,512 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 76c4ff0c848e5ffb5ad9989537063596: 2024-11-26T19:14:04,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(8855): Flush requested on 76c4ff0c848e5ffb5ad9989537063596 2024-11-26T19:14:04,645 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-26T19:14:04,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. 2024-11-26T19:14:04,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. as already flushing 2024-11-26T19:14:04,646 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 76c4ff0c848e5ffb5ad9989537063596 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-26T19:14:04,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. 2024-11-26T19:14:04,646 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:04,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:04,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c4ff0c848e5ffb5ad9989537063596, store=A 2024-11-26T19:14:04,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:14:04,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c4ff0c848e5ffb5ad9989537063596, store=B 2024-11-26T19:14:04,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:14:04,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c4ff0c848e5ffb5ad9989537063596, store=C 2024-11-26T19:14:04,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:14:04,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:04,653 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/A/9039de866ce0411381a247749e6b0b34 is 50, key is test_row_0/A:col10/1732648444645/Put/seqid=0 2024-11-26T19:14:04,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741912_1088 (size=14741) 2024-11-26T19:14:04,676 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=383 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/A/9039de866ce0411381a247749e6b0b34 2024-11-26T19:14:04,690 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/B/17034839721943708a7042d0da70a8ef is 50, key is test_row_0/B:col10/1732648444645/Put/seqid=0 2024-11-26T19:14:04,705 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:04,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49198 deadline: 1732648504693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:04,706 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:04,706 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:04,706 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:04,707 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:04,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49182 deadline: 1732648504699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:04,707 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:04,707 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:04,707 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:04,707 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:04,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49132 deadline: 1732648504702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:04,708 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:04,708 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:04,708 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:04,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741913_1089 (size=12301) 2024-11-26T19:14:04,804 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-26T19:14:04,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. 2024-11-26T19:14:04,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. as already flushing 2024-11-26T19:14:04,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. 2024-11-26T19:14:04,805 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:04,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:04,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:04,817 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:04,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49198 deadline: 1732648504812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:04,817 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:04,817 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:04,818 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:04,818 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:04,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49182 deadline: 1732648504812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:04,818 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:04,818 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:04,818 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:04,818 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:04,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49132 deadline: 1732648504812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:04,819 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:04,819 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:04,819 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:04,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-26T19:14:04,958 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-26T19:14:04,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. 2024-11-26T19:14:04,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. as already flushing 2024-11-26T19:14:04,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. 2024-11-26T19:14:04,959 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:04,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:04,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:05,024 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:05,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49198 deadline: 1732648505022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:05,024 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:05,024 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:05,024 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:05,025 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:05,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49182 deadline: 1732648505022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:05,025 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:05,025 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:05,025 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:05,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49132 deadline: 1732648505022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:05,025 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:05,026 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:05,026 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:05,026 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:05,113 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-26T19:14:05,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. 2024-11-26T19:14:05,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. as already flushing 2024-11-26T19:14:05,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. 2024-11-26T19:14:05,113 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:05,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:05,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:05,121 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=383 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/B/17034839721943708a7042d0da70a8ef 2024-11-26T19:14:05,133 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/C/c7f85a14ae9649b092eb8a86f94adc8e is 50, key is test_row_0/C:col10/1732648444645/Put/seqid=0 2024-11-26T19:14:05,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741914_1090 (size=12301) 2024-11-26T19:14:05,267 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-26T19:14:05,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. 2024-11-26T19:14:05,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. as already flushing 2024-11-26T19:14:05,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. 2024-11-26T19:14:05,267 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:05,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:05,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:05,337 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:05,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49198 deadline: 1732648505332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:05,338 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:05,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49182 deadline: 1732648505333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:05,338 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:05,338 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:05,339 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:05,339 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:05,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49132 deadline: 1732648505333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:05,339 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:05,339 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:05,340 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:05,340 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:05,340 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:05,340 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:05,421 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-26T19:14:05,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. 2024-11-26T19:14:05,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. as already flushing 2024-11-26T19:14:05,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. 2024-11-26T19:14:05,422 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:05,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:05,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:05,544 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=383 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/C/c7f85a14ae9649b092eb8a86f94adc8e 2024-11-26T19:14:05,555 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/A/9039de866ce0411381a247749e6b0b34 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/9039de866ce0411381a247749e6b0b34 2024-11-26T19:14:05,561 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/9039de866ce0411381a247749e6b0b34, entries=200, sequenceid=383, filesize=14.4 K 2024-11-26T19:14:05,562 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/B/17034839721943708a7042d0da70a8ef as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/17034839721943708a7042d0da70a8ef 2024-11-26T19:14:05,571 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/17034839721943708a7042d0da70a8ef, entries=150, sequenceid=383, filesize=12.0 K 2024-11-26T19:14:05,572 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/C/c7f85a14ae9649b092eb8a86f94adc8e as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/c7f85a14ae9649b092eb8a86f94adc8e 2024-11-26T19:14:05,575 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-26T19:14:05,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. 2024-11-26T19:14:05,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. as already flushing 2024-11-26T19:14:05,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. 2024-11-26T19:14:05,576 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:05,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:05,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:05,580 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/c7f85a14ae9649b092eb8a86f94adc8e, entries=150, sequenceid=383, filesize=12.0 K 2024-11-26T19:14:05,581 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 76c4ff0c848e5ffb5ad9989537063596 in 935ms, sequenceid=383, compaction requested=true 2024-11-26T19:14:05,581 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 76c4ff0c848e5ffb5ad9989537063596: 2024-11-26T19:14:05,581 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T19:14:05,581 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 76c4ff0c848e5ffb5ad9989537063596:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T19:14:05,581 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:14:05,582 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T19:14:05,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 76c4ff0c848e5ffb5ad9989537063596:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T19:14:05,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:14:05,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 76c4ff0c848e5ffb5ad9989537063596:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T19:14:05,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T19:14:05,582 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42567 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T19:14:05,582 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1541): 76c4ff0c848e5ffb5ad9989537063596/A is initiating minor compaction (all files) 2024-11-26T19:14:05,583 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 76c4ff0c848e5ffb5ad9989537063596/A in TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. 2024-11-26T19:14:05,583 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/2e4e6f7645ad44c583b88cf3d2644c69, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/58dc31274336443fb6448a3bb609ce64, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/9039de866ce0411381a247749e6b0b34] into tmpdir=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp, totalSize=41.6 K 2024-11-26T19:14:05,583 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T19:14:05,583 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1541): 76c4ff0c848e5ffb5ad9989537063596/B is initiating minor compaction (all files) 2024-11-26T19:14:05,583 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 76c4ff0c848e5ffb5ad9989537063596/B in TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. 2024-11-26T19:14:05,583 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/478b3346f9f842da835187dcdf211b46, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/8bfed9f8d1dd41f392250f028c36e710, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/17034839721943708a7042d0da70a8ef] into tmpdir=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp, totalSize=36.8 K 2024-11-26T19:14:05,584 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2e4e6f7645ad44c583b88cf3d2644c69, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1732648443732 2024-11-26T19:14:05,584 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting 478b3346f9f842da835187dcdf211b46, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1732648443732 2024-11-26T19:14:05,584 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting 58dc31274336443fb6448a3bb609ce64, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1732648443842 2024-11-26T19:14:05,584 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting 8bfed9f8d1dd41f392250f028c36e710, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1732648443855 2024-11-26T19:14:05,603 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9039de866ce0411381a247749e6b0b34, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=383, earliestPutTs=1732648444002 2024-11-26T19:14:05,603 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting 17034839721943708a7042d0da70a8ef, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=383, earliestPutTs=1732648444002 2024-11-26T19:14:05,622 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 76c4ff0c848e5ffb5ad9989537063596#B#compaction#78 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T19:14:05,623 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/B/7beb3a2a241847abb903b2be47e0ba8b is 50, key is test_row_0/B:col10/1732648444645/Put/seqid=0 2024-11-26T19:14:05,626 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 76c4ff0c848e5ffb5ad9989537063596#A#compaction#79 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T19:14:05,627 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/A/9f1999d5e47f4b5cbe558c6c9f05e42e is 50, key is test_row_0/A:col10/1732648444645/Put/seqid=0 2024-11-26T19:14:05,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741915_1091 (size=13187) 2024-11-26T19:14:05,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741916_1092 (size=13187) 2024-11-26T19:14:05,654 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/B/7beb3a2a241847abb903b2be47e0ba8b as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/7beb3a2a241847abb903b2be47e0ba8b 2024-11-26T19:14:05,662 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 76c4ff0c848e5ffb5ad9989537063596/B of 76c4ff0c848e5ffb5ad9989537063596 into 7beb3a2a241847abb903b2be47e0ba8b(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T19:14:05,662 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 76c4ff0c848e5ffb5ad9989537063596: 2024-11-26T19:14:05,662 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., storeName=76c4ff0c848e5ffb5ad9989537063596/B, priority=13, startTime=1732648445581; duration=0sec 2024-11-26T19:14:05,662 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T19:14:05,662 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 76c4ff0c848e5ffb5ad9989537063596:B 2024-11-26T19:14:05,662 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T19:14:05,663 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T19:14:05,663 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1541): 76c4ff0c848e5ffb5ad9989537063596/C is initiating minor compaction (all files) 2024-11-26T19:14:05,664 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 76c4ff0c848e5ffb5ad9989537063596/C in TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. 2024-11-26T19:14:05,664 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/6fab4748f5394c0eb9109fb7f84ecbaf, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/9adcb6f4199242698dfb6b3e69bdac63, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/c7f85a14ae9649b092eb8a86f94adc8e] into tmpdir=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp, totalSize=36.8 K 2024-11-26T19:14:05,664 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting 6fab4748f5394c0eb9109fb7f84ecbaf, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1732648443732 2024-11-26T19:14:05,665 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting 9adcb6f4199242698dfb6b3e69bdac63, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1732648443855 2024-11-26T19:14:05,665 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting c7f85a14ae9649b092eb8a86f94adc8e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=383, earliestPutTs=1732648444002 2024-11-26T19:14:05,690 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 76c4ff0c848e5ffb5ad9989537063596#C#compaction#80 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T19:14:05,691 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/C/555abfe58b9a418f925cdfa01c312335 is 50, key is test_row_0/C:col10/1732648444645/Put/seqid=0 2024-11-26T19:14:05,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741917_1093 (size=13187) 2024-11-26T19:14:05,732 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-26T19:14:05,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. 2024-11-26T19:14:05,732 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing 76c4ff0c848e5ffb5ad9989537063596 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-26T19:14:05,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c4ff0c848e5ffb5ad9989537063596, store=A 2024-11-26T19:14:05,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:14:05,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c4ff0c848e5ffb5ad9989537063596, store=B 2024-11-26T19:14:05,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:14:05,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c4ff0c848e5ffb5ad9989537063596, store=C 2024-11-26T19:14:05,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:14:05,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/A/95f8b1c195774ea2b6046f4879b2e956 is 50, key is test_row_0/A:col10/1732648444690/Put/seqid=0 2024-11-26T19:14:05,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741918_1094 (size=12301) 2024-11-26T19:14:05,772 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=403 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/A/95f8b1c195774ea2b6046f4879b2e956 2024-11-26T19:14:05,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/B/4fae14e2a5174a329f8a829156d972ba is 50, key is test_row_0/B:col10/1732648444690/Put/seqid=0 2024-11-26T19:14:05,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741919_1095 (size=12301) 2024-11-26T19:14:05,828 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=403 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/B/4fae14e2a5174a329f8a829156d972ba 2024-11-26T19:14:05,849 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. as already flushing 2024-11-26T19:14:05,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(8855): Flush requested on 76c4ff0c848e5ffb5ad9989537063596 2024-11-26T19:14:05,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/C/cbd4e5db7e5f4f9b9d4535ba227b9eed is 50, key is test_row_0/C:col10/1732648444690/Put/seqid=0 2024-11-26T19:14:05,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-26T19:14:05,882 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:05,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49182 deadline: 1732648505879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:05,883 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:05,883 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:05,883 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:05,884 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:05,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49132 deadline: 1732648505881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:05,885 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:05,885 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:05,885 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:05,889 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:05,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49198 deadline: 1732648505882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:05,889 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:05,890 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:05,890 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:05,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741920_1096 (size=12301) 2024-11-26T19:14:05,899 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=403 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/C/cbd4e5db7e5f4f9b9d4535ba227b9eed 2024-11-26T19:14:05,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/A/95f8b1c195774ea2b6046f4879b2e956 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/95f8b1c195774ea2b6046f4879b2e956 2024-11-26T19:14:05,914 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/95f8b1c195774ea2b6046f4879b2e956, entries=150, sequenceid=403, filesize=12.0 K 2024-11-26T19:14:05,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/B/4fae14e2a5174a329f8a829156d972ba as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/4fae14e2a5174a329f8a829156d972ba 2024-11-26T19:14:05,924 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/4fae14e2a5174a329f8a829156d972ba, entries=150, sequenceid=403, filesize=12.0 K 2024-11-26T19:14:05,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/C/cbd4e5db7e5f4f9b9d4535ba227b9eed as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/cbd4e5db7e5f4f9b9d4535ba227b9eed 2024-11-26T19:14:05,933 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/cbd4e5db7e5f4f9b9d4535ba227b9eed, entries=150, sequenceid=403, filesize=12.0 K 2024-11-26T19:14:05,934 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 76c4ff0c848e5ffb5ad9989537063596 in 202ms, sequenceid=403, compaction requested=false 2024-11-26T19:14:05,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for 76c4ff0c848e5ffb5ad9989537063596: 2024-11-26T19:14:05,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. 2024-11-26T19:14:05,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-26T19:14:05,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-26T19:14:05,940 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-26T19:14:05,940 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2180 sec 2024-11-26T19:14:05,945 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 2.2280 sec 2024-11-26T19:14:05,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(8855): Flush requested on 76c4ff0c848e5ffb5ad9989537063596 2024-11-26T19:14:05,995 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 76c4ff0c848e5ffb5ad9989537063596 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-26T19:14:05,995 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c4ff0c848e5ffb5ad9989537063596, store=A 2024-11-26T19:14:05,995 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:14:05,995 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c4ff0c848e5ffb5ad9989537063596, store=B 2024-11-26T19:14:05,995 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:14:05,995 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c4ff0c848e5ffb5ad9989537063596, store=C 2024-11-26T19:14:05,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:14:06,002 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/A/c28f600cf9a8477c887850a9d28b8170 is 50, key is test_row_0/A:col10/1732648445881/Put/seqid=0 2024-11-26T19:14:06,011 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741921_1097 (size=12301) 2024-11-26T19:14:06,013 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=417 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/A/c28f600cf9a8477c887850a9d28b8170 2024-11-26T19:14:06,029 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/B/d839b1e9763d42ef82dce75b0a8ff051 is 50, key is test_row_0/B:col10/1732648445881/Put/seqid=0 2024-11-26T19:14:06,030 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:06,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49198 deadline: 1732648506027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:06,030 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:06,030 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:06,030 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:06,033 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:06,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49132 deadline: 1732648506029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:06,034 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:06,034 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:06,034 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:06,034 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:06,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49182 deadline: 1732648506032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:06,034 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:06,034 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:06,034 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:06,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741922_1098 (size=12301) 2024-11-26T19:14:06,052 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/A/9f1999d5e47f4b5cbe558c6c9f05e42e as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/9f1999d5e47f4b5cbe558c6c9f05e42e 2024-11-26T19:14:06,061 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 76c4ff0c848e5ffb5ad9989537063596/A of 76c4ff0c848e5ffb5ad9989537063596 into 9f1999d5e47f4b5cbe558c6c9f05e42e(size=12.9 K), total size for store is 24.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T19:14:06,061 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 76c4ff0c848e5ffb5ad9989537063596: 2024-11-26T19:14:06,061 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., storeName=76c4ff0c848e5ffb5ad9989537063596/A, priority=13, startTime=1732648445581; duration=0sec 2024-11-26T19:14:06,061 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:14:06,062 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 76c4ff0c848e5ffb5ad9989537063596:A 2024-11-26T19:14:06,117 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/C/555abfe58b9a418f925cdfa01c312335 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/555abfe58b9a418f925cdfa01c312335 2024-11-26T19:14:06,136 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 76c4ff0c848e5ffb5ad9989537063596/C of 76c4ff0c848e5ffb5ad9989537063596 into 555abfe58b9a418f925cdfa01c312335(size=12.9 K), total size for store is 24.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T19:14:06,137 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 76c4ff0c848e5ffb5ad9989537063596: 2024-11-26T19:14:06,137 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., storeName=76c4ff0c848e5ffb5ad9989537063596/C, priority=13, startTime=1732648445582; duration=0sec 2024-11-26T19:14:06,137 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:14:06,137 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 76c4ff0c848e5ffb5ad9989537063596:C 2024-11-26T19:14:06,138 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:06,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49198 deadline: 1732648506133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:06,139 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:06,139 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:06,139 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:06,143 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:06,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49182 deadline: 1732648506142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:06,144 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:06,144 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:06,144 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:06,151 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:06,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49132 deadline: 1732648506144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:06,152 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:06,152 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:06,152 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:06,356 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:06,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49198 deadline: 1732648506352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:06,356 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:06,356 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:06,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49182 deadline: 1732648506352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:06,356 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:06,356 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:06,357 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:06,357 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:06,357 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:06,363 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:06,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49132 deadline: 1732648506361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:06,364 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:06,364 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:06,364 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:06,445 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=417 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/B/d839b1e9763d42ef82dce75b0a8ff051 2024-11-26T19:14:06,455 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/C/14cbdcd5e9e24fe1bb94dfdcc261d946 is 50, key is test_row_0/C:col10/1732648445881/Put/seqid=0 2024-11-26T19:14:06,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741923_1099 (size=12301) 2024-11-26T19:14:06,667 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:06,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49198 deadline: 1732648506662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:06,667 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:06,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49182 deadline: 1732648506662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:06,668 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:06,668 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:06,668 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:06,669 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:06,669 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:06,669 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:06,674 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:06,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49132 deadline: 1732648506671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:06,674 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:06,675 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:06,675 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:06,860 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=417 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/C/14cbdcd5e9e24fe1bb94dfdcc261d946 2024-11-26T19:14:06,867 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/A/c28f600cf9a8477c887850a9d28b8170 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/c28f600cf9a8477c887850a9d28b8170 2024-11-26T19:14:06,875 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/c28f600cf9a8477c887850a9d28b8170, entries=150, sequenceid=417, filesize=12.0 K 2024-11-26T19:14:06,876 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/B/d839b1e9763d42ef82dce75b0a8ff051 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/d839b1e9763d42ef82dce75b0a8ff051 2024-11-26T19:14:06,882 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/d839b1e9763d42ef82dce75b0a8ff051, entries=150, sequenceid=417, filesize=12.0 K 2024-11-26T19:14:06,884 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/C/14cbdcd5e9e24fe1bb94dfdcc261d946 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/14cbdcd5e9e24fe1bb94dfdcc261d946 2024-11-26T19:14:06,893 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/14cbdcd5e9e24fe1bb94dfdcc261d946, entries=150, sequenceid=417, filesize=12.0 K 2024-11-26T19:14:06,894 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 76c4ff0c848e5ffb5ad9989537063596 in 899ms, sequenceid=417, compaction requested=true 2024-11-26T19:14:06,895 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 76c4ff0c848e5ffb5ad9989537063596: 2024-11-26T19:14:06,895 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T19:14:06,895 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 76c4ff0c848e5ffb5ad9989537063596:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T19:14:06,895 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:14:06,895 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T19:14:06,897 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T19:14:06,897 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1541): 76c4ff0c848e5ffb5ad9989537063596/B is initiating minor compaction (all files) 2024-11-26T19:14:06,898 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 76c4ff0c848e5ffb5ad9989537063596/B in TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. 2024-11-26T19:14:06,898 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/7beb3a2a241847abb903b2be47e0ba8b, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/4fae14e2a5174a329f8a829156d972ba, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/d839b1e9763d42ef82dce75b0a8ff051] into tmpdir=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp, totalSize=36.9 K 2024-11-26T19:14:06,898 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T19:14:06,898 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1541): 76c4ff0c848e5ffb5ad9989537063596/A is initiating minor compaction (all files) 2024-11-26T19:14:06,898 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 76c4ff0c848e5ffb5ad9989537063596/A in TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. 2024-11-26T19:14:06,898 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/9f1999d5e47f4b5cbe558c6c9f05e42e, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/95f8b1c195774ea2b6046f4879b2e956, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/c28f600cf9a8477c887850a9d28b8170] into tmpdir=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp, totalSize=36.9 K 2024-11-26T19:14:06,898 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting 7beb3a2a241847abb903b2be47e0ba8b, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=383, earliestPutTs=1732648444002 2024-11-26T19:14:06,895 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 76c4ff0c848e5ffb5ad9989537063596:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T19:14:06,898 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:14:06,899 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 76c4ff0c848e5ffb5ad9989537063596:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T19:14:06,899 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T19:14:06,899 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9f1999d5e47f4b5cbe558c6c9f05e42e, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=383, earliestPutTs=1732648444002 2024-11-26T19:14:06,899 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting 4fae14e2a5174a329f8a829156d972ba, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=403, earliestPutTs=1732648444673 2024-11-26T19:14:06,899 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting d839b1e9763d42ef82dce75b0a8ff051, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=417, earliestPutTs=1732648445878 2024-11-26T19:14:06,899 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting 95f8b1c195774ea2b6046f4879b2e956, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=403, earliestPutTs=1732648444673 2024-11-26T19:14:06,900 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting c28f600cf9a8477c887850a9d28b8170, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=417, earliestPutTs=1732648445878 2024-11-26T19:14:06,911 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 76c4ff0c848e5ffb5ad9989537063596#A#compaction#87 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T19:14:06,912 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/A/727f69e616fb4b7882fa905a42b8caa6 is 50, key is test_row_0/A:col10/1732648445881/Put/seqid=0 2024-11-26T19:14:06,914 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 76c4ff0c848e5ffb5ad9989537063596#B#compaction#88 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T19:14:06,915 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/B/d5f694571dc4445dbe3751638267516d is 50, key is test_row_0/B:col10/1732648445881/Put/seqid=0 2024-11-26T19:14:06,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741924_1100 (size=13289) 2024-11-26T19:14:06,927 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741925_1101 (size=13289) 2024-11-26T19:14:07,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(8855): Flush requested on 76c4ff0c848e5ffb5ad9989537063596 2024-11-26T19:14:07,175 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 76c4ff0c848e5ffb5ad9989537063596 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-26T19:14:07,175 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c4ff0c848e5ffb5ad9989537063596, store=A 2024-11-26T19:14:07,176 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:14:07,176 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c4ff0c848e5ffb5ad9989537063596, store=B 2024-11-26T19:14:07,176 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:14:07,176 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c4ff0c848e5ffb5ad9989537063596, store=C 2024-11-26T19:14:07,176 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:14:07,185 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/A/0be2854a865847a2ac6eca3e05b63214 is 50, key is test_row_0/A:col10/1732648447173/Put/seqid=0 2024-11-26T19:14:07,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741926_1102 (size=14741) 2024-11-26T19:14:07,212 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=438 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/A/0be2854a865847a2ac6eca3e05b63214 2024-11-26T19:14:07,231 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/B/88f099863a1d4349b1948e92b9003dd8 is 50, key is test_row_0/B:col10/1732648447173/Put/seqid=0 2024-11-26T19:14:07,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741927_1103 (size=12301) 2024-11-26T19:14:07,238 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=438 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/B/88f099863a1d4349b1948e92b9003dd8 2024-11-26T19:14:07,248 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/C/c3c9b130a90342c2976b23f0821de4b4 is 50, key is test_row_0/C:col10/1732648447173/Put/seqid=0 2024-11-26T19:14:07,261 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:07,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49198 deadline: 1732648507255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:07,262 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:07,262 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:07,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741928_1104 (size=12301) 2024-11-26T19:14:07,262 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:07,263 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:07,264 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=438 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/C/c3c9b130a90342c2976b23f0821de4b4 2024-11-26T19:14:07,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49132 deadline: 1732648507255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:07,264 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:07,264 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:07,264 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:07,266 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:07,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49182 deadline: 1732648507261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:07,268 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:07,268 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:07,268 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:07,277 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/A/0be2854a865847a2ac6eca3e05b63214 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/0be2854a865847a2ac6eca3e05b63214 2024-11-26T19:14:07,282 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/0be2854a865847a2ac6eca3e05b63214, entries=200, sequenceid=438, filesize=14.4 K 2024-11-26T19:14:07,283 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/B/88f099863a1d4349b1948e92b9003dd8 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/88f099863a1d4349b1948e92b9003dd8 2024-11-26T19:14:07,289 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/88f099863a1d4349b1948e92b9003dd8, entries=150, sequenceid=438, filesize=12.0 K 2024-11-26T19:14:07,291 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/C/c3c9b130a90342c2976b23f0821de4b4 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/c3c9b130a90342c2976b23f0821de4b4 2024-11-26T19:14:07,299 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/c3c9b130a90342c2976b23f0821de4b4, entries=150, sequenceid=438, filesize=12.0 K 2024-11-26T19:14:07,301 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=80.51 KB/82440 for 76c4ff0c848e5ffb5ad9989537063596 in 126ms, sequenceid=438, compaction requested=true 2024-11-26T19:14:07,302 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 76c4ff0c848e5ffb5ad9989537063596: 2024-11-26T19:14:07,302 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 76c4ff0c848e5ffb5ad9989537063596:A, priority=-2147483648, current under compaction store size is 3 2024-11-26T19:14:07,302 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-26T19:14:07,302 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 76c4ff0c848e5ffb5ad9989537063596:B, priority=-2147483648, current under compaction store size is 3 2024-11-26T19:14:07,302 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-26T19:14:07,302 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 76c4ff0c848e5ffb5ad9989537063596:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T19:14:07,302 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-11-26T19:14:07,333 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/A/727f69e616fb4b7882fa905a42b8caa6 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/727f69e616fb4b7882fa905a42b8caa6 2024-11-26T19:14:07,346 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/B/d5f694571dc4445dbe3751638267516d as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/d5f694571dc4445dbe3751638267516d 2024-11-26T19:14:07,349 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 76c4ff0c848e5ffb5ad9989537063596/A of 76c4ff0c848e5ffb5ad9989537063596 into 727f69e616fb4b7882fa905a42b8caa6(size=13.0 K), total size for store is 27.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T19:14:07,349 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 76c4ff0c848e5ffb5ad9989537063596: 2024-11-26T19:14:07,349 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., storeName=76c4ff0c848e5ffb5ad9989537063596/A, priority=13, startTime=1732648446895; duration=0sec 2024-11-26T19:14:07,349 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-11-26T19:14:07,349 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 76c4ff0c848e5ffb5ad9989537063596:A 2024-11-26T19:14:07,349 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 76c4ff0c848e5ffb5ad9989537063596:A 2024-11-26T19:14:07,349 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T19:14:07,351 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50090 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T19:14:07,352 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1541): 76c4ff0c848e5ffb5ad9989537063596/C is initiating minor compaction (all files) 2024-11-26T19:14:07,352 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 76c4ff0c848e5ffb5ad9989537063596/C in TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. 2024-11-26T19:14:07,352 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/555abfe58b9a418f925cdfa01c312335, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/cbd4e5db7e5f4f9b9d4535ba227b9eed, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/14cbdcd5e9e24fe1bb94dfdcc261d946, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/c3c9b130a90342c2976b23f0821de4b4] into tmpdir=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp, totalSize=48.9 K 2024-11-26T19:14:07,352 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting 555abfe58b9a418f925cdfa01c312335, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=383, earliestPutTs=1732648444002 2024-11-26T19:14:07,353 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting cbd4e5db7e5f4f9b9d4535ba227b9eed, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=403, earliestPutTs=1732648444673 2024-11-26T19:14:07,353 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting 14cbdcd5e9e24fe1bb94dfdcc261d946, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=417, earliestPutTs=1732648445878 2024-11-26T19:14:07,354 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting c3c9b130a90342c2976b23f0821de4b4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=438, earliestPutTs=1732648446022 2024-11-26T19:14:07,357 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 76c4ff0c848e5ffb5ad9989537063596/B of 76c4ff0c848e5ffb5ad9989537063596 into d5f694571dc4445dbe3751638267516d(size=13.0 K), total size for store is 25.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T19:14:07,357 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 76c4ff0c848e5ffb5ad9989537063596: 2024-11-26T19:14:07,357 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., storeName=76c4ff0c848e5ffb5ad9989537063596/B, priority=13, startTime=1732648446895; duration=0sec 2024-11-26T19:14:07,357 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-26T19:14:07,357 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 76c4ff0c848e5ffb5ad9989537063596:B 2024-11-26T19:14:07,358 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 76c4ff0c848e5ffb5ad9989537063596:B 2024-11-26T19:14:07,358 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 4 compacting, 0 eligible, 16 blocking 2024-11-26T19:14:07,358 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-26T19:14:07,358 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-26T19:14:07,358 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. because compaction request was cancelled 2024-11-26T19:14:07,358 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 76c4ff0c848e5ffb5ad9989537063596:C 2024-11-26T19:14:07,358 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-26T19:14:07,359 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-26T19:14:07,359 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-26T19:14:07,359 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. because compaction request was cancelled 2024-11-26T19:14:07,359 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 76c4ff0c848e5ffb5ad9989537063596:B 2024-11-26T19:14:07,359 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-26T19:14:07,361 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-26T19:14:07,361 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-26T19:14:07,361 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. because compaction request was cancelled 2024-11-26T19:14:07,361 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 76c4ff0c848e5ffb5ad9989537063596:A 2024-11-26T19:14:07,370 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 76c4ff0c848e5ffb5ad9989537063596#C#compaction#92 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T19:14:07,371 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/C/c1a1ded1570e40b3a3b0374e98b0b035 is 50, key is test_row_0/C:col10/1732648447173/Put/seqid=0 2024-11-26T19:14:07,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(8855): Flush requested on 76c4ff0c848e5ffb5ad9989537063596 2024-11-26T19:14:07,374 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 76c4ff0c848e5ffb5ad9989537063596 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-26T19:14:07,374 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c4ff0c848e5ffb5ad9989537063596, store=A 2024-11-26T19:14:07,375 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:14:07,375 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c4ff0c848e5ffb5ad9989537063596, store=B 2024-11-26T19:14:07,375 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:14:07,375 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c4ff0c848e5ffb5ad9989537063596, store=C 2024-11-26T19:14:07,375 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:14:07,390 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/A/3c7760f9639448418a3942f3bafced8b is 50, key is test_row_0/A:col10/1732648447373/Put/seqid=0 2024-11-26T19:14:07,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741929_1105 (size=13323) 2024-11-26T19:14:07,416 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:07,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49198 deadline: 1732648507409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:07,417 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:07,417 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:07,417 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:07,417 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:07,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49132 deadline: 1732648507410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:07,418 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:07,418 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:07,418 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:07,417 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:07,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49182 deadline: 1732648507412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:07,418 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:07,419 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:07,419 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:07,422 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/C/c1a1ded1570e40b3a3b0374e98b0b035 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/c1a1ded1570e40b3a3b0374e98b0b035 2024-11-26T19:14:07,429 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 76c4ff0c848e5ffb5ad9989537063596/C of 76c4ff0c848e5ffb5ad9989537063596 into c1a1ded1570e40b3a3b0374e98b0b035(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T19:14:07,430 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 76c4ff0c848e5ffb5ad9989537063596: 2024-11-26T19:14:07,430 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., storeName=76c4ff0c848e5ffb5ad9989537063596/C, priority=12, startTime=1732648447302; duration=0sec 2024-11-26T19:14:07,430 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:14:07,430 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 76c4ff0c848e5ffb5ad9989537063596:C 2024-11-26T19:14:07,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741930_1106 (size=12301) 2024-11-26T19:14:07,524 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:07,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49198 deadline: 1732648507523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:07,525 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:07,525 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:07,525 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:07,525 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:07,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49182 deadline: 1732648507523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:07,526 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:07,526 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:07,526 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:07,526 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:07,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49132 deadline: 1732648507524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:07,527 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:07,527 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:07,527 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:07,734 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:07,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49132 deadline: 1732648507733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:07,734 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:07,734 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:07,734 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:07,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49198 deadline: 1732648507733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:07,735 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:07,735 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:07,735 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:07,735 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:07,735 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:07,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49182 deadline: 1732648507733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:07,735 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:07,735 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:07,735 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:07,831 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=456 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/A/3c7760f9639448418a3942f3bafced8b 2024-11-26T19:14:07,840 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/B/8212b6b252d04625a8f51140872aafcc is 50, key is test_row_0/B:col10/1732648447373/Put/seqid=0 2024-11-26T19:14:07,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741931_1107 (size=12301) 2024-11-26T19:14:07,858 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=456 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/B/8212b6b252d04625a8f51140872aafcc 2024-11-26T19:14:07,867 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/C/77f2fd7654044f27b1014fc28c1cd7e4 is 50, key is test_row_0/C:col10/1732648447373/Put/seqid=0 2024-11-26T19:14:07,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-26T19:14:07,873 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T19:14:07,876 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T19:14:07,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-11-26T19:14:07,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-26T19:14:07,879 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T19:14:07,881 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T19:14:07,881 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T19:14:07,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741932_1108 (size=12301) 2024-11-26T19:14:07,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-26T19:14:08,033 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-26T19:14:08,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. 2024-11-26T19:14:08,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. as already flushing 2024-11-26T19:14:08,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. 2024-11-26T19:14:08,034 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:08,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:08,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:08,044 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:08,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49132 deadline: 1732648508043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:08,045 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:08,045 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:08,045 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:08,045 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:08,045 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:08,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49198 deadline: 1732648508044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:08,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49182 deadline: 1732648508044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:08,046 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:08,046 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:08,046 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:08,046 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:08,046 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:08,046 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:08,188 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-26T19:14:08,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. 2024-11-26T19:14:08,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. as already flushing 2024-11-26T19:14:08,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. 2024-11-26T19:14:08,189 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:08,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:08,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:08,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-26T19:14:08,288 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=456 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/C/77f2fd7654044f27b1014fc28c1cd7e4 2024-11-26T19:14:08,295 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/A/3c7760f9639448418a3942f3bafced8b as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/3c7760f9639448418a3942f3bafced8b 2024-11-26T19:14:08,302 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/3c7760f9639448418a3942f3bafced8b, entries=150, sequenceid=456, filesize=12.0 K 2024-11-26T19:14:08,303 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/B/8212b6b252d04625a8f51140872aafcc as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/8212b6b252d04625a8f51140872aafcc 2024-11-26T19:14:08,308 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/8212b6b252d04625a8f51140872aafcc, entries=150, sequenceid=456, filesize=12.0 K 2024-11-26T19:14:08,310 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/C/77f2fd7654044f27b1014fc28c1cd7e4 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/77f2fd7654044f27b1014fc28c1cd7e4 2024-11-26T19:14:08,324 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/77f2fd7654044f27b1014fc28c1cd7e4, entries=150, sequenceid=456, filesize=12.0 K 2024-11-26T19:14:08,325 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 76c4ff0c848e5ffb5ad9989537063596 in 951ms, sequenceid=456, compaction requested=true 2024-11-26T19:14:08,325 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 76c4ff0c848e5ffb5ad9989537063596: 2024-11-26T19:14:08,325 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 76c4ff0c848e5ffb5ad9989537063596:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T19:14:08,326 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:14:08,326 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T19:14:08,326 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 76c4ff0c848e5ffb5ad9989537063596:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T19:14:08,326 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T19:14:08,326 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 76c4ff0c848e5ffb5ad9989537063596:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T19:14:08,326 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-26T19:14:08,326 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T19:14:08,327 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40331 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T19:14:08,327 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1541): 76c4ff0c848e5ffb5ad9989537063596/A is initiating minor compaction (all files) 2024-11-26T19:14:08,328 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 76c4ff0c848e5ffb5ad9989537063596/A in TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. 2024-11-26T19:14:08,328 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T19:14:08,328 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/727f69e616fb4b7882fa905a42b8caa6, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/0be2854a865847a2ac6eca3e05b63214, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/3c7760f9639448418a3942f3bafced8b] into tmpdir=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp, totalSize=39.4 K 2024-11-26T19:14:08,328 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1541): 76c4ff0c848e5ffb5ad9989537063596/B is initiating minor compaction (all files) 2024-11-26T19:14:08,328 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 76c4ff0c848e5ffb5ad9989537063596/B in TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. 2024-11-26T19:14:08,328 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/d5f694571dc4445dbe3751638267516d, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/88f099863a1d4349b1948e92b9003dd8, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/8212b6b252d04625a8f51140872aafcc] into tmpdir=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp, totalSize=37.0 K 2024-11-26T19:14:08,330 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting 727f69e616fb4b7882fa905a42b8caa6, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=417, earliestPutTs=1732648445878 2024-11-26T19:14:08,330 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting d5f694571dc4445dbe3751638267516d, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=417, earliestPutTs=1732648445878 2024-11-26T19:14:08,330 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0be2854a865847a2ac6eca3e05b63214, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=438, earliestPutTs=1732648446022 2024-11-26T19:14:08,331 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting 88f099863a1d4349b1948e92b9003dd8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=438, earliestPutTs=1732648446022 2024-11-26T19:14:08,331 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3c7760f9639448418a3942f3bafced8b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=456, earliestPutTs=1732648447216 2024-11-26T19:14:08,331 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting 8212b6b252d04625a8f51140872aafcc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=456, earliestPutTs=1732648447216 2024-11-26T19:14:08,344 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-26T19:14:08,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. 2024-11-26T19:14:08,344 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing 76c4ff0c848e5ffb5ad9989537063596 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-26T19:14:08,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c4ff0c848e5ffb5ad9989537063596, store=A 2024-11-26T19:14:08,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:14:08,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c4ff0c848e5ffb5ad9989537063596, store=B 2024-11-26T19:14:08,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:14:08,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c4ff0c848e5ffb5ad9989537063596, store=C 2024-11-26T19:14:08,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:14:08,349 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 76c4ff0c848e5ffb5ad9989537063596#A#compaction#96 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T19:14:08,349 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/A/22f35ae29aea445397cc31c99f303396 is 50, key is test_row_0/A:col10/1732648447373/Put/seqid=0 2024-11-26T19:14:08,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/A/9a0a1bc8870e4bba8eccf4c73a25d618 is 50, key is test_row_0/A:col10/1732648447399/Put/seqid=0 2024-11-26T19:14:08,377 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 76c4ff0c848e5ffb5ad9989537063596#B#compaction#98 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T19:14:08,377 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/B/cc8eef064d834d02bd014dc4d582f8bc is 50, key is test_row_0/B:col10/1732648447373/Put/seqid=0 2024-11-26T19:14:08,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741933_1109 (size=13391) 2024-11-26T19:14:08,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741934_1110 (size=12301) 2024-11-26T19:14:08,402 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=473 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/A/9a0a1bc8870e4bba8eccf4c73a25d618 2024-11-26T19:14:08,412 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/A/22f35ae29aea445397cc31c99f303396 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/22f35ae29aea445397cc31c99f303396 2024-11-26T19:14:08,421 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 76c4ff0c848e5ffb5ad9989537063596/A of 76c4ff0c848e5ffb5ad9989537063596 into 22f35ae29aea445397cc31c99f303396(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T19:14:08,421 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 76c4ff0c848e5ffb5ad9989537063596: 2024-11-26T19:14:08,421 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., storeName=76c4ff0c848e5ffb5ad9989537063596/A, priority=13, startTime=1732648448325; duration=0sec 2024-11-26T19:14:08,422 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T19:14:08,422 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 76c4ff0c848e5ffb5ad9989537063596:A 2024-11-26T19:14:08,422 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-26T19:14:08,423 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-26T19:14:08,423 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-26T19:14:08,423 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. because compaction request was cancelled 2024-11-26T19:14:08,423 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 76c4ff0c848e5ffb5ad9989537063596:C 2024-11-26T19:14:08,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/B/2b8f3e209cb24ca38c377b95cc86782b is 50, key is test_row_0/B:col10/1732648447399/Put/seqid=0 2024-11-26T19:14:08,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741935_1111 (size=13391) 2024-11-26T19:14:08,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741936_1112 (size=12301) 2024-11-26T19:14:08,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-26T19:14:08,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(8855): Flush requested on 76c4ff0c848e5ffb5ad9989537063596 2024-11-26T19:14:08,556 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. as already flushing 2024-11-26T19:14:08,619 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:08,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49198 deadline: 1732648508610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:08,620 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:08,620 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:08,620 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:08,625 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:08,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49182 deadline: 1732648508618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:08,626 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:08,626 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:08,626 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:08,627 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:08,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49132 deadline: 1732648508618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:08,627 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:08,627 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:08,627 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:08,738 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:08,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49198 deadline: 1732648508732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:08,739 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:08,739 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:08,739 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:08,739 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:08,739 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:08,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49182 deadline: 1732648508732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:08,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49132 deadline: 1732648508732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:08,740 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:08,740 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:08,740 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:08,740 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:08,740 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:08,740 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:08,839 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=473 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/B/2b8f3e209cb24ca38c377b95cc86782b 2024-11-26T19:14:08,846 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/B/cc8eef064d834d02bd014dc4d582f8bc as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/cc8eef064d834d02bd014dc4d582f8bc 2024-11-26T19:14:08,856 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 76c4ff0c848e5ffb5ad9989537063596/B of 76c4ff0c848e5ffb5ad9989537063596 into cc8eef064d834d02bd014dc4d582f8bc(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T19:14:08,856 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 76c4ff0c848e5ffb5ad9989537063596: 2024-11-26T19:14:08,856 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., storeName=76c4ff0c848e5ffb5ad9989537063596/B, priority=13, startTime=1732648448326; duration=0sec 2024-11-26T19:14:08,856 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:14:08,856 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 76c4ff0c848e5ffb5ad9989537063596:B 2024-11-26T19:14:08,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/C/7156b4db90524dd39986fc8a74732339 is 50, key is test_row_0/C:col10/1732648447399/Put/seqid=0 2024-11-26T19:14:08,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741937_1113 (size=12301) 2024-11-26T19:14:08,943 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:08,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49198 deadline: 1732648508942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:08,943 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:08,944 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:08,944 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:08,953 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:08,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49132 deadline: 1732648508952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:08,954 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:08,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49182 deadline: 1732648508952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:08,955 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:08,955 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:08,955 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:08,956 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:08,956 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:08,956 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:09,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-26T19:14:09,254 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:09,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49198 deadline: 1732648509252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:09,255 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:09,255 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:09,255 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:09,264 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:09,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49182 deadline: 1732648509262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:09,267 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:09,267 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:09,267 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:09,267 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:09,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49132 deadline: 1732648509262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:09,267 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:09,268 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:09,268 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:09,294 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=473 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/C/7156b4db90524dd39986fc8a74732339 2024-11-26T19:14:09,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/A/9a0a1bc8870e4bba8eccf4c73a25d618 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/9a0a1bc8870e4bba8eccf4c73a25d618 2024-11-26T19:14:09,315 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/9a0a1bc8870e4bba8eccf4c73a25d618, entries=150, sequenceid=473, filesize=12.0 K 2024-11-26T19:14:09,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/B/2b8f3e209cb24ca38c377b95cc86782b as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/2b8f3e209cb24ca38c377b95cc86782b 2024-11-26T19:14:09,327 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/2b8f3e209cb24ca38c377b95cc86782b, entries=150, sequenceid=473, filesize=12.0 K 2024-11-26T19:14:09,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/C/7156b4db90524dd39986fc8a74732339 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/7156b4db90524dd39986fc8a74732339 2024-11-26T19:14:09,334 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/7156b4db90524dd39986fc8a74732339, entries=150, sequenceid=473, filesize=12.0 K 2024-11-26T19:14:09,337 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=93.93 KB/96180 for 76c4ff0c848e5ffb5ad9989537063596 in 993ms, sequenceid=473, compaction requested=true 2024-11-26T19:14:09,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for 76c4ff0c848e5ffb5ad9989537063596: 2024-11-26T19:14:09,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. 2024-11-26T19:14:09,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-11-26T19:14:09,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-11-26T19:14:09,343 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-26T19:14:09,343 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4580 sec 2024-11-26T19:14:09,345 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 1.4680 sec 2024-11-26T19:14:09,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(8855): Flush requested on 76c4ff0c848e5ffb5ad9989537063596 2024-11-26T19:14:09,764 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 76c4ff0c848e5ffb5ad9989537063596 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-26T19:14:09,765 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c4ff0c848e5ffb5ad9989537063596, store=A 2024-11-26T19:14:09,765 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:14:09,765 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c4ff0c848e5ffb5ad9989537063596, store=B 2024-11-26T19:14:09,765 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:14:09,765 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c4ff0c848e5ffb5ad9989537063596, store=C 2024-11-26T19:14:09,765 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:14:09,786 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/A/685ce3166285452491597a3710aec249 is 50, key is test_row_0/A:col10/1732648449762/Put/seqid=0 2024-11-26T19:14:09,810 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:09,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49198 deadline: 1732648509803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:09,812 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:09,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49132 deadline: 1732648509806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:09,812 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:09,812 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:09,812 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:09,812 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:09,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49182 deadline: 1732648509807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:09,812 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:09,812 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:09,813 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:09,813 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:09,813 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:09,813 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:09,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741938_1114 (size=14741) 2024-11-26T19:14:09,823 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=493 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/A/685ce3166285452491597a3710aec249 2024-11-26T19:14:09,850 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/B/00b9e53947b74b6c8a3344aa957bb211 is 50, key is test_row_0/B:col10/1732648449762/Put/seqid=0 2024-11-26T19:14:09,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741939_1115 (size=12301) 2024-11-26T19:14:09,861 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=493 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/B/00b9e53947b74b6c8a3344aa957bb211 2024-11-26T19:14:09,874 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/C/799a44e1cfe3493c833f2d3046f4ceb6 is 50, key is test_row_0/C:col10/1732648449762/Put/seqid=0 2024-11-26T19:14:09,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741940_1116 (size=12301) 2024-11-26T19:14:09,923 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:09,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49198 deadline: 1732648509922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:09,924 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:09,924 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:09,924 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:09,924 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:09,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49132 deadline: 1732648509922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:09,924 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:09,924 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:09,924 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:09,924 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:09,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49182 deadline: 1732648509922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:09,925 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:09,925 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:09,925 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:10,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-26T19:14:10,023 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T19:14:10,023 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T19:14:10,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-11-26T19:14:10,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-26T19:14:10,027 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T19:14:10,028 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T19:14:10,028 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T19:14:10,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-26T19:14:10,134 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:10,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49182 deadline: 1732648510133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:10,135 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:10,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49198 deadline: 1732648510133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:10,135 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:10,135 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:10,135 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:10,135 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:10,135 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:10,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49132 deadline: 1732648510133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:10,136 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:10,136 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:10,136 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:10,136 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:10,136 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:10,181 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-26T19:14:10,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. 2024-11-26T19:14:10,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. as already flushing 2024-11-26T19:14:10,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. 2024-11-26T19:14:10,181 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:10,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:10,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:10,281 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=493 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/C/799a44e1cfe3493c833f2d3046f4ceb6 2024-11-26T19:14:10,290 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/A/685ce3166285452491597a3710aec249 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/685ce3166285452491597a3710aec249 2024-11-26T19:14:10,296 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/685ce3166285452491597a3710aec249, entries=200, sequenceid=493, filesize=14.4 K 2024-11-26T19:14:10,304 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/B/00b9e53947b74b6c8a3344aa957bb211 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/00b9e53947b74b6c8a3344aa957bb211 2024-11-26T19:14:10,319 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/00b9e53947b74b6c8a3344aa957bb211, entries=150, sequenceid=493, filesize=12.0 K 2024-11-26T19:14:10,321 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/C/799a44e1cfe3493c833f2d3046f4ceb6 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/799a44e1cfe3493c833f2d3046f4ceb6 2024-11-26T19:14:10,335 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-26T19:14:10,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. 2024-11-26T19:14:10,336 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/799a44e1cfe3493c833f2d3046f4ceb6, entries=150, sequenceid=493, filesize=12.0 K 2024-11-26T19:14:10,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. as already flushing 2024-11-26T19:14:10,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. 2024-11-26T19:14:10,336 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:10,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:10,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:10,339 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 76c4ff0c848e5ffb5ad9989537063596 in 575ms, sequenceid=493, compaction requested=true 2024-11-26T19:14:10,339 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 76c4ff0c848e5ffb5ad9989537063596: 2024-11-26T19:14:10,340 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 76c4ff0c848e5ffb5ad9989537063596:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T19:14:10,340 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:14:10,340 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T19:14:10,340 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T19:14:10,340 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 76c4ff0c848e5ffb5ad9989537063596:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T19:14:10,340 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:14:10,340 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 76c4ff0c848e5ffb5ad9989537063596:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T19:14:10,340 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T19:14:10,341 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40433 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T19:14:10,341 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37993 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T19:14:10,342 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1541): 76c4ff0c848e5ffb5ad9989537063596/A is initiating minor compaction (all files) 2024-11-26T19:14:10,342 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1541): 76c4ff0c848e5ffb5ad9989537063596/B is initiating minor compaction (all files) 2024-11-26T19:14:10,342 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 76c4ff0c848e5ffb5ad9989537063596/A in TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. 2024-11-26T19:14:10,342 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 76c4ff0c848e5ffb5ad9989537063596/B in TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. 2024-11-26T19:14:10,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-26T19:14:10,342 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/22f35ae29aea445397cc31c99f303396, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/9a0a1bc8870e4bba8eccf4c73a25d618, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/685ce3166285452491597a3710aec249] into tmpdir=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp, totalSize=39.5 K 2024-11-26T19:14:10,342 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/cc8eef064d834d02bd014dc4d582f8bc, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/2b8f3e209cb24ca38c377b95cc86782b, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/00b9e53947b74b6c8a3344aa957bb211] into tmpdir=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp, totalSize=37.1 K 2024-11-26T19:14:10,342 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting cc8eef064d834d02bd014dc4d582f8bc, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=456, earliestPutTs=1732648447216 2024-11-26T19:14:10,342 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting 22f35ae29aea445397cc31c99f303396, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=456, earliestPutTs=1732648447216 2024-11-26T19:14:10,343 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9a0a1bc8870e4bba8eccf4c73a25d618, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=473, earliestPutTs=1732648447399 2024-11-26T19:14:10,343 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting 2b8f3e209cb24ca38c377b95cc86782b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=473, earliestPutTs=1732648447399 2024-11-26T19:14:10,343 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting 685ce3166285452491597a3710aec249, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=493, earliestPutTs=1732648448602 2024-11-26T19:14:10,344 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting 00b9e53947b74b6c8a3344aa957bb211, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=493, earliestPutTs=1732648448607 2024-11-26T19:14:10,355 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 76c4ff0c848e5ffb5ad9989537063596#B#compaction#104 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T19:14:10,355 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 76c4ff0c848e5ffb5ad9989537063596#A#compaction#105 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T19:14:10,355 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/B/eea9951c5e1045ce8384227a507535a6 is 50, key is test_row_0/B:col10/1732648449762/Put/seqid=0 2024-11-26T19:14:10,356 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/A/1b4c2ad639fa4c29b1bf71bd8d9e61b0 is 50, key is test_row_0/A:col10/1732648449762/Put/seqid=0 2024-11-26T19:14:10,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741941_1117 (size=13493) 2024-11-26T19:14:10,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741942_1118 (size=13493) 2024-11-26T19:14:10,378 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/A/1b4c2ad639fa4c29b1bf71bd8d9e61b0 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/1b4c2ad639fa4c29b1bf71bd8d9e61b0 2024-11-26T19:14:10,395 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 76c4ff0c848e5ffb5ad9989537063596/A of 76c4ff0c848e5ffb5ad9989537063596 into 1b4c2ad639fa4c29b1bf71bd8d9e61b0(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T19:14:10,395 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 76c4ff0c848e5ffb5ad9989537063596: 2024-11-26T19:14:10,395 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., storeName=76c4ff0c848e5ffb5ad9989537063596/A, priority=13, startTime=1732648450340; duration=0sec 2024-11-26T19:14:10,395 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T19:14:10,395 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 76c4ff0c848e5ffb5ad9989537063596:A 2024-11-26T19:14:10,395 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T19:14:10,397 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50226 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T19:14:10,397 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1541): 76c4ff0c848e5ffb5ad9989537063596/C is initiating minor compaction (all files) 2024-11-26T19:14:10,397 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 76c4ff0c848e5ffb5ad9989537063596/C in TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. 2024-11-26T19:14:10,398 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/c1a1ded1570e40b3a3b0374e98b0b035, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/77f2fd7654044f27b1014fc28c1cd7e4, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/7156b4db90524dd39986fc8a74732339, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/799a44e1cfe3493c833f2d3046f4ceb6] into tmpdir=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp, totalSize=49.0 K 2024-11-26T19:14:10,401 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting c1a1ded1570e40b3a3b0374e98b0b035, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=438, earliestPutTs=1732648446022 2024-11-26T19:14:10,401 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting 77f2fd7654044f27b1014fc28c1cd7e4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=456, earliestPutTs=1732648447216 2024-11-26T19:14:10,401 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7156b4db90524dd39986fc8a74732339, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=473, earliestPutTs=1732648447399 2024-11-26T19:14:10,402 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting 799a44e1cfe3493c833f2d3046f4ceb6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=493, earliestPutTs=1732648448607 2024-11-26T19:14:10,439 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 76c4ff0c848e5ffb5ad9989537063596#C#compaction#106 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T19:14:10,440 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/C/29edc9700d0a4a2696107f67660f393e is 50, key is test_row_0/C:col10/1732648449762/Put/seqid=0 2024-11-26T19:14:10,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(8855): Flush requested on 76c4ff0c848e5ffb5ad9989537063596 2024-11-26T19:14:10,448 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 76c4ff0c848e5ffb5ad9989537063596 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-26T19:14:10,451 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c4ff0c848e5ffb5ad9989537063596, store=A 2024-11-26T19:14:10,451 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:14:10,451 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c4ff0c848e5ffb5ad9989537063596, store=B 2024-11-26T19:14:10,452 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:14:10,452 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c4ff0c848e5ffb5ad9989537063596, store=C 2024-11-26T19:14:10,452 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:14:10,464 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/A/c6371fdd17a644b3b4e96fad3e4392ed is 50, key is test_row_0/A:col10/1732648450447/Put/seqid=0 2024-11-26T19:14:10,478 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-26T19:14:10,478 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T19:14:10,479 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-26T19:14:10,479 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T19:14:10,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741943_1119 (size=13459) 2024-11-26T19:14:10,481 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-26T19:14:10,481 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T19:14:10,482 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-26T19:14:10,482 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T19:14:10,484 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:14:10,484 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:14:10,484 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:14:10,485 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:14:10,485 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T19:14:10,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741944_1120 (size=9857) 2024-11-26T19:14:10,486 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-26T19:14:10,486 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T19:14:10,486 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:14:10,486 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-26T19:14:10,486 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:14:10,486 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:14:10,486 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T19:14:10,486 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:14:10,486 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:14:10,486 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T19:14:10,486 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T19:14:10,486 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T19:14:10,486 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:14:10,486 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T19:14:10,487 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:14:10,487 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:14:10,487 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T19:14:10,489 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-26T19:14:10,489 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T19:14:10,489 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:14:10,489 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:14:10,489 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T19:14:10,489 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-26T19:14:10,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. 2024-11-26T19:14:10,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. as already flushing 2024-11-26T19:14:10,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. 2024-11-26T19:14:10,490 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:10,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:10,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:10,500 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/C/29edc9700d0a4a2696107f67660f393e as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/29edc9700d0a4a2696107f67660f393e 2024-11-26T19:14:10,518 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 76c4ff0c848e5ffb5ad9989537063596/C of 76c4ff0c848e5ffb5ad9989537063596 into 29edc9700d0a4a2696107f67660f393e(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T19:14:10,518 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 76c4ff0c848e5ffb5ad9989537063596: 2024-11-26T19:14:10,518 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., storeName=76c4ff0c848e5ffb5ad9989537063596/C, priority=12, startTime=1732648450340; duration=0sec 2024-11-26T19:14:10,518 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:14:10,518 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 76c4ff0c848e5ffb5ad9989537063596:C 2024-11-26T19:14:10,643 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-26T19:14:10,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. 2024-11-26T19:14:10,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. as already flushing 2024-11-26T19:14:10,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. 2024-11-26T19:14:10,644 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:10,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:10,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:10,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-26T19:14:10,775 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/B/eea9951c5e1045ce8384227a507535a6 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/eea9951c5e1045ce8384227a507535a6 2024-11-26T19:14:10,782 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 76c4ff0c848e5ffb5ad9989537063596/B of 76c4ff0c848e5ffb5ad9989537063596 into eea9951c5e1045ce8384227a507535a6(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T19:14:10,782 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 76c4ff0c848e5ffb5ad9989537063596: 2024-11-26T19:14:10,782 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., storeName=76c4ff0c848e5ffb5ad9989537063596/B, priority=13, startTime=1732648450340; duration=0sec 2024-11-26T19:14:10,783 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:14:10,783 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 76c4ff0c848e5ffb5ad9989537063596:B 2024-11-26T19:14:10,797 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-26T19:14:10,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. 2024-11-26T19:14:10,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. as already flushing 2024-11-26T19:14:10,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. 2024-11-26T19:14:10,798 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:10,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:10,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:10,885 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=510 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/A/c6371fdd17a644b3b4e96fad3e4392ed 2024-11-26T19:14:10,902 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/B/6cc150e9c331431f9fa9223e98ae91fb is 50, key is test_row_0/B:col10/1732648450447/Put/seqid=0 2024-11-26T19:14:10,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741945_1121 (size=9857) 2024-11-26T19:14:10,953 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-26T19:14:10,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. 2024-11-26T19:14:10,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. as already flushing 2024-11-26T19:14:10,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. 2024-11-26T19:14:10,953 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:10,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:10,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:11,107 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-26T19:14:11,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. 2024-11-26T19:14:11,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. as already flushing 2024-11-26T19:14:11,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. 2024-11-26T19:14:11,107 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:11,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:11,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:11,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-26T19:14:11,260 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-26T19:14:11,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. 2024-11-26T19:14:11,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. as already flushing 2024-11-26T19:14:11,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. 2024-11-26T19:14:11,261 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:11,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:11,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:11,308 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=510 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/B/6cc150e9c331431f9fa9223e98ae91fb 2024-11-26T19:14:11,316 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/C/51bb91aa53734b9a83c429b96e91ac50 is 50, key is test_row_0/C:col10/1732648450447/Put/seqid=0 2024-11-26T19:14:11,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741946_1122 (size=9857) 2024-11-26T19:14:11,320 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=510 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/C/51bb91aa53734b9a83c429b96e91ac50 2024-11-26T19:14:11,325 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/A/c6371fdd17a644b3b4e96fad3e4392ed as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/c6371fdd17a644b3b4e96fad3e4392ed 2024-11-26T19:14:11,330 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/c6371fdd17a644b3b4e96fad3e4392ed, entries=100, sequenceid=510, filesize=9.6 K 2024-11-26T19:14:11,331 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/B/6cc150e9c331431f9fa9223e98ae91fb as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/6cc150e9c331431f9fa9223e98ae91fb 2024-11-26T19:14:11,335 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/6cc150e9c331431f9fa9223e98ae91fb, entries=100, sequenceid=510, filesize=9.6 K 2024-11-26T19:14:11,336 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/C/51bb91aa53734b9a83c429b96e91ac50 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/51bb91aa53734b9a83c429b96e91ac50 2024-11-26T19:14:11,340 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/51bb91aa53734b9a83c429b96e91ac50, entries=100, sequenceid=510, filesize=9.6 K 2024-11-26T19:14:11,341 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=80.51 KB/82440 for 76c4ff0c848e5ffb5ad9989537063596 in 894ms, sequenceid=510, compaction requested=false 2024-11-26T19:14:11,341 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 76c4ff0c848e5ffb5ad9989537063596: 2024-11-26T19:14:11,413 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-26T19:14:11,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. 2024-11-26T19:14:11,414 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing 76c4ff0c848e5ffb5ad9989537063596 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-26T19:14:11,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c4ff0c848e5ffb5ad9989537063596, store=A 2024-11-26T19:14:11,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:14:11,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c4ff0c848e5ffb5ad9989537063596, store=B 2024-11-26T19:14:11,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:14:11,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c4ff0c848e5ffb5ad9989537063596, store=C 2024-11-26T19:14:11,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:14:11,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/A/b0ab0ab0b9c649f7a3b42088c99e8dd5 is 50, key is test_row_0/A:col10/1732648450455/Put/seqid=0 2024-11-26T19:14:11,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741947_1123 (size=12301) 2024-11-26T19:14:11,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(8855): Flush requested on 76c4ff0c848e5ffb5ad9989537063596 2024-11-26T19:14:11,553 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. as already flushing 2024-11-26T19:14:11,554 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-26T19:14:11,554 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T19:14:11,554 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:14:11,554 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:14:11,554 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T19:14:11,593 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-26T19:14:11,593 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T19:14:11,593 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:14:11,593 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:14:11,593 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T19:14:11,823 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=527 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/A/b0ab0ab0b9c649f7a3b42088c99e8dd5 2024-11-26T19:14:11,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/B/e82d431962f1464b9f1cb67e504b72a1 is 50, key is test_row_0/B:col10/1732648450455/Put/seqid=0 2024-11-26T19:14:11,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741948_1124 (size=12301) 2024-11-26T19:14:12,134 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-26T19:14:12,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-26T19:14:12,235 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=527 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/B/e82d431962f1464b9f1cb67e504b72a1 2024-11-26T19:14:12,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/C/6ea70f98984e4c2e950357dd6d639bb3 is 50, key is test_row_0/C:col10/1732648450455/Put/seqid=0 2024-11-26T19:14:12,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741949_1125 (size=12301) 2024-11-26T19:14:12,648 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=527 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/C/6ea70f98984e4c2e950357dd6d639bb3 2024-11-26T19:14:12,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/A/b0ab0ab0b9c649f7a3b42088c99e8dd5 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/b0ab0ab0b9c649f7a3b42088c99e8dd5 2024-11-26T19:14:12,658 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/b0ab0ab0b9c649f7a3b42088c99e8dd5, entries=150, sequenceid=527, filesize=12.0 K 2024-11-26T19:14:12,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/B/e82d431962f1464b9f1cb67e504b72a1 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/e82d431962f1464b9f1cb67e504b72a1 2024-11-26T19:14:12,663 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/e82d431962f1464b9f1cb67e504b72a1, entries=150, sequenceid=527, filesize=12.0 K 2024-11-26T19:14:12,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/C/6ea70f98984e4c2e950357dd6d639bb3 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/6ea70f98984e4c2e950357dd6d639bb3 2024-11-26T19:14:12,667 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/6ea70f98984e4c2e950357dd6d639bb3, entries=150, sequenceid=527, filesize=12.0 K 2024-11-26T19:14:12,668 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=13.42 KB/13740 for 76c4ff0c848e5ffb5ad9989537063596 in 1254ms, sequenceid=527, compaction requested=true 2024-11-26T19:14:12,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for 76c4ff0c848e5ffb5ad9989537063596: 2024-11-26T19:14:12,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. 2024-11-26T19:14:12,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-11-26T19:14:12,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-11-26T19:14:12,672 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-26T19:14:12,672 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6420 sec 2024-11-26T19:14:12,674 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 2.6500 sec 2024-11-26T19:14:14,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-26T19:14:14,192 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T19:14:14,192 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-26T19:14:14,192 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 20 2024-11-26T19:14:14,192 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 129 2024-11-26T19:14:14,192 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 17 2024-11-26T19:14:14,192 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 111 2024-11-26T19:14:14,192 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 138 2024-11-26T19:14:14,192 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-26T19:14:14,192 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2657 2024-11-26T19:14:14,193 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2597 2024-11-26T19:14:14,193 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-26T19:14:14,193 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1181 2024-11-26T19:14:14,193 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3526 rows 2024-11-26T19:14:14,193 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1189 2024-11-26T19:14:14,193 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3544 rows 2024-11-26T19:14:14,193 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-26T19:14:14,193 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-26T19:14:14,193 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:14:14,193 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:14:14,193 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T19:14:14,197 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-26T19:14:14,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-26T19:14:14,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-26T19:14:14,204 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732648454204"}]},"ts":"1732648454204"} 2024-11-26T19:14:14,206 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-26T19:14:14,206 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-26T19:14:14,207 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-26T19:14:14,211 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=76c4ff0c848e5ffb5ad9989537063596, UNASSIGN}] 2024-11-26T19:14:14,212 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=76c4ff0c848e5ffb5ad9989537063596, UNASSIGN 2024-11-26T19:14:14,213 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=76c4ff0c848e5ffb5ad9989537063596, regionState=CLOSING, regionLocation=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:14,215 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=76c4ff0c848e5ffb5ad9989537063596, UNASSIGN because future has completed 2024-11-26T19:14:14,215 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-26T19:14:14,215 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE, hasLock=false; CloseRegionProcedure 76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970}] 2024-11-26T19:14:14,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-26T19:14:14,371 INFO [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] handler.UnassignRegionHandler(122): Close 76c4ff0c848e5ffb5ad9989537063596 2024-11-26T19:14:14,372 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-26T19:14:14,372 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1722): Closing 76c4ff0c848e5ffb5ad9989537063596, disabling compactions & flushes 2024-11-26T19:14:14,372 INFO [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. 2024-11-26T19:14:14,372 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. 2024-11-26T19:14:14,373 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. after waiting 0 ms 2024-11-26T19:14:14,373 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. 2024-11-26T19:14:14,373 INFO [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(2902): Flushing 76c4ff0c848e5ffb5ad9989537063596 3/3 column families, dataSize=13.42 KB heapSize=40.59 KB 2024-11-26T19:14:14,373 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c4ff0c848e5ffb5ad9989537063596, store=A 2024-11-26T19:14:14,373 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:14:14,373 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c4ff0c848e5ffb5ad9989537063596, store=B 2024-11-26T19:14:14,373 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:14:14,373 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c4ff0c848e5ffb5ad9989537063596, store=C 2024-11-26T19:14:14,373 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:14:14,377 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/A/72b52c71508a4ab3ad218a451224dfd6 is 50, key is test_row_0/A:col10/1732648451592/Put/seqid=0 2024-11-26T19:14:14,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741950_1126 (size=9857) 2024-11-26T19:14:14,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-26T19:14:14,782 INFO [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=532 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/A/72b52c71508a4ab3ad218a451224dfd6 2024-11-26T19:14:14,789 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/B/91ecbfad536748c2855865b072661ba5 is 50, key is test_row_0/B:col10/1732648451592/Put/seqid=0 2024-11-26T19:14:14,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741951_1127 (size=9857) 2024-11-26T19:14:14,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-26T19:14:15,194 INFO [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=532 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/B/91ecbfad536748c2855865b072661ba5 2024-11-26T19:14:15,201 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/C/3c49ff008bb24ba3924208365b34dbb9 is 50, key is test_row_0/C:col10/1732648451592/Put/seqid=0 2024-11-26T19:14:15,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741952_1128 (size=9857) 2024-11-26T19:14:15,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-26T19:14:15,606 INFO [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=532 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/C/3c49ff008bb24ba3924208365b34dbb9 2024-11-26T19:14:15,611 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/A/72b52c71508a4ab3ad218a451224dfd6 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/72b52c71508a4ab3ad218a451224dfd6 2024-11-26T19:14:15,615 INFO [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/72b52c71508a4ab3ad218a451224dfd6, entries=100, sequenceid=532, filesize=9.6 K 2024-11-26T19:14:15,616 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/B/91ecbfad536748c2855865b072661ba5 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/91ecbfad536748c2855865b072661ba5 2024-11-26T19:14:15,619 INFO [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/91ecbfad536748c2855865b072661ba5, entries=100, sequenceid=532, filesize=9.6 K 2024-11-26T19:14:15,620 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/.tmp/C/3c49ff008bb24ba3924208365b34dbb9 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/3c49ff008bb24ba3924208365b34dbb9 2024-11-26T19:14:15,624 INFO [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/3c49ff008bb24ba3924208365b34dbb9, entries=100, sequenceid=532, filesize=9.6 K 2024-11-26T19:14:15,625 INFO [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for 76c4ff0c848e5ffb5ad9989537063596 in 1252ms, sequenceid=532, compaction requested=true 2024-11-26T19:14:15,625 DEBUG [StoreCloser-TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/a198223cb57a4570b8cb1128e123f979, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/eab18453588b45ddad179336a0d7ccd4, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/1143addfdb464bfbaf1996658cd75569, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/417e2ebbceda48289466048b374abb09, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/a059261dcd974c5d8eed5b1d29328488, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/03c3e8b4bbb54f09acf353e917df4e86, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/bce7dd392f5c4c1db67aab71d936efec, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/35f3765d522a4a5cb13ee27baeea2c1b, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/88f933f70dd24dca965172f1945c0a78, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/1e60651003db49adad7dbf608e7764ce, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/de1be7379cca41919698005e01bc443d, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/139e2b3b8af743bfbecc48835a1357dc, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/128c2dc5a4704ea6ac55fce0184eb846, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/6c9a6be2cdd34d3e80c6d012d0bddb33, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/23ad685edd2d4041813aec8a7b6602ef, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/71f2b009b2244b68ad864e49663a98cc, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/77d0a0ab017d44e9b68da403b4454e10, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/a97c2c7078f346d4866eb711c11b0796, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/d24b66db5ff54903a97bf0b7d04b52a0, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/6b65df8e384447379e3b1e045e6846e5, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/9cc0e8b7ca2341f9a661a7c6af090fa6, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/5b1c7d96ac094fe8a988e5dc7ea8f379, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/421ad5e1c9c84d8684b7ef8e6d6d76f8, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/2e4e6f7645ad44c583b88cf3d2644c69, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/58dc31274336443fb6448a3bb609ce64, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/9039de866ce0411381a247749e6b0b34, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/9f1999d5e47f4b5cbe558c6c9f05e42e, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/95f8b1c195774ea2b6046f4879b2e956, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/727f69e616fb4b7882fa905a42b8caa6, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/c28f600cf9a8477c887850a9d28b8170, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/0be2854a865847a2ac6eca3e05b63214, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/22f35ae29aea445397cc31c99f303396, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/3c7760f9639448418a3942f3bafced8b, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/9a0a1bc8870e4bba8eccf4c73a25d618, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/685ce3166285452491597a3710aec249] to archive 2024-11-26T19:14:15,628 DEBUG [StoreCloser-TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-26T19:14:15,633 DEBUG [StoreCloser-TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/a198223cb57a4570b8cb1128e123f979 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/a198223cb57a4570b8cb1128e123f979 2024-11-26T19:14:15,635 DEBUG [StoreCloser-TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/eab18453588b45ddad179336a0d7ccd4 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/eab18453588b45ddad179336a0d7ccd4 2024-11-26T19:14:15,636 DEBUG [StoreCloser-TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/1143addfdb464bfbaf1996658cd75569 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/1143addfdb464bfbaf1996658cd75569 2024-11-26T19:14:15,637 DEBUG [StoreCloser-TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/417e2ebbceda48289466048b374abb09 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/417e2ebbceda48289466048b374abb09 2024-11-26T19:14:15,639 DEBUG [StoreCloser-TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/a059261dcd974c5d8eed5b1d29328488 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/a059261dcd974c5d8eed5b1d29328488 2024-11-26T19:14:15,640 DEBUG [StoreCloser-TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/03c3e8b4bbb54f09acf353e917df4e86 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/03c3e8b4bbb54f09acf353e917df4e86 2024-11-26T19:14:15,641 DEBUG [StoreCloser-TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/bce7dd392f5c4c1db67aab71d936efec to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/bce7dd392f5c4c1db67aab71d936efec 2024-11-26T19:14:15,642 DEBUG [StoreCloser-TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/35f3765d522a4a5cb13ee27baeea2c1b to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/35f3765d522a4a5cb13ee27baeea2c1b 2024-11-26T19:14:15,644 DEBUG [StoreCloser-TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/88f933f70dd24dca965172f1945c0a78 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/88f933f70dd24dca965172f1945c0a78 2024-11-26T19:14:15,645 DEBUG [StoreCloser-TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/1e60651003db49adad7dbf608e7764ce to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/1e60651003db49adad7dbf608e7764ce 2024-11-26T19:14:15,646 DEBUG [StoreCloser-TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/de1be7379cca41919698005e01bc443d to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/de1be7379cca41919698005e01bc443d 2024-11-26T19:14:15,647 DEBUG [StoreCloser-TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/139e2b3b8af743bfbecc48835a1357dc to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/139e2b3b8af743bfbecc48835a1357dc 2024-11-26T19:14:15,649 DEBUG [StoreCloser-TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/128c2dc5a4704ea6ac55fce0184eb846 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/128c2dc5a4704ea6ac55fce0184eb846 2024-11-26T19:14:15,650 DEBUG [StoreCloser-TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/6c9a6be2cdd34d3e80c6d012d0bddb33 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/6c9a6be2cdd34d3e80c6d012d0bddb33 2024-11-26T19:14:15,651 DEBUG [StoreCloser-TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/23ad685edd2d4041813aec8a7b6602ef to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/23ad685edd2d4041813aec8a7b6602ef 2024-11-26T19:14:15,652 DEBUG [StoreCloser-TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/71f2b009b2244b68ad864e49663a98cc to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/71f2b009b2244b68ad864e49663a98cc 2024-11-26T19:14:15,653 DEBUG [StoreCloser-TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/77d0a0ab017d44e9b68da403b4454e10 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/77d0a0ab017d44e9b68da403b4454e10 2024-11-26T19:14:15,654 DEBUG [StoreCloser-TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/a97c2c7078f346d4866eb711c11b0796 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/a97c2c7078f346d4866eb711c11b0796 2024-11-26T19:14:15,655 DEBUG [StoreCloser-TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/d24b66db5ff54903a97bf0b7d04b52a0 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/d24b66db5ff54903a97bf0b7d04b52a0 2024-11-26T19:14:15,656 DEBUG [StoreCloser-TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/6b65df8e384447379e3b1e045e6846e5 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/6b65df8e384447379e3b1e045e6846e5 2024-11-26T19:14:15,658 DEBUG [StoreCloser-TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/9cc0e8b7ca2341f9a661a7c6af090fa6 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/9cc0e8b7ca2341f9a661a7c6af090fa6 2024-11-26T19:14:15,659 DEBUG [StoreCloser-TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/5b1c7d96ac094fe8a988e5dc7ea8f379 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/5b1c7d96ac094fe8a988e5dc7ea8f379 2024-11-26T19:14:15,660 DEBUG [StoreCloser-TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/421ad5e1c9c84d8684b7ef8e6d6d76f8 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/421ad5e1c9c84d8684b7ef8e6d6d76f8 2024-11-26T19:14:15,661 DEBUG [StoreCloser-TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/2e4e6f7645ad44c583b88cf3d2644c69 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/2e4e6f7645ad44c583b88cf3d2644c69 2024-11-26T19:14:15,662 DEBUG [StoreCloser-TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/58dc31274336443fb6448a3bb609ce64 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/58dc31274336443fb6448a3bb609ce64 2024-11-26T19:14:15,663 DEBUG [StoreCloser-TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/9039de866ce0411381a247749e6b0b34 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/9039de866ce0411381a247749e6b0b34 2024-11-26T19:14:15,665 DEBUG [StoreCloser-TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/9f1999d5e47f4b5cbe558c6c9f05e42e to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/9f1999d5e47f4b5cbe558c6c9f05e42e 2024-11-26T19:14:15,666 DEBUG [StoreCloser-TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/95f8b1c195774ea2b6046f4879b2e956 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/95f8b1c195774ea2b6046f4879b2e956 2024-11-26T19:14:15,667 DEBUG [StoreCloser-TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/727f69e616fb4b7882fa905a42b8caa6 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/727f69e616fb4b7882fa905a42b8caa6 2024-11-26T19:14:15,668 DEBUG [StoreCloser-TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/c28f600cf9a8477c887850a9d28b8170 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/c28f600cf9a8477c887850a9d28b8170 2024-11-26T19:14:15,669 DEBUG [StoreCloser-TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/0be2854a865847a2ac6eca3e05b63214 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/0be2854a865847a2ac6eca3e05b63214 2024-11-26T19:14:15,670 DEBUG [StoreCloser-TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/22f35ae29aea445397cc31c99f303396 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/22f35ae29aea445397cc31c99f303396 2024-11-26T19:14:15,671 DEBUG [StoreCloser-TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/3c7760f9639448418a3942f3bafced8b to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/3c7760f9639448418a3942f3bafced8b 2024-11-26T19:14:15,672 DEBUG [StoreCloser-TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/9a0a1bc8870e4bba8eccf4c73a25d618 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/9a0a1bc8870e4bba8eccf4c73a25d618 2024-11-26T19:14:15,673 DEBUG [StoreCloser-TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/685ce3166285452491597a3710aec249 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/685ce3166285452491597a3710aec249 2024-11-26T19:14:15,687 DEBUG [StoreCloser-TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/caee9bb5427744818d43a7ad8a612201, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/ad268253061e43b88ec365621129766a, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/c42911a70e754a3eb876673cbd4e84fa, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/84313ca47744468780bcf3b82b5296af, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/2d4a10652e2f46a5ac9b71d9b13c322d, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/af438f13b1204e9e8acfc13f307c89d4, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/753c8c618f1f4d4bb636e3d4bb699e98, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/a8898f8302174564861f2109f1015483, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/c4a0c5cf1f9a4ffcb4b3afd52bb65774, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/8963dd6b3d994966883672717578324d, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/e340a0fc862e4e6899c9ba3cce88c90d, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/7b79c14e7270430095e6008dcaeece65, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/674882adcdff4f50bb75443483598cef, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/725f293dbdb644f6b1a8146d0fcd5831, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/f9e4dc2be254487292a3465fa91e842d, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/13106d68607942e687f8bb2f54fa1a93, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/59f264b3fff84b77822329d7c684aa42, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/8dfb3e1b0884467ba576198f7f349403, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/da15be7eff4449a1807670adea92ff11, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/d41bfc4dd3d348028897b897e60c3af3, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/ff81f2d8e01346e1b7b68b1892e6f8fb, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/73626e1537194c10bfe236f3d7188198, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/478b3346f9f842da835187dcdf211b46, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/32f6f58cb60847a5983a76e858f37cdd, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/8bfed9f8d1dd41f392250f028c36e710, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/7beb3a2a241847abb903b2be47e0ba8b, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/17034839721943708a7042d0da70a8ef, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/4fae14e2a5174a329f8a829156d972ba, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/d5f694571dc4445dbe3751638267516d, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/d839b1e9763d42ef82dce75b0a8ff051, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/88f099863a1d4349b1948e92b9003dd8, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/cc8eef064d834d02bd014dc4d582f8bc, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/8212b6b252d04625a8f51140872aafcc, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/2b8f3e209cb24ca38c377b95cc86782b, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/00b9e53947b74b6c8a3344aa957bb211] to archive 2024-11-26T19:14:15,687 DEBUG [StoreCloser-TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-26T19:14:15,689 DEBUG [StoreCloser-TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/caee9bb5427744818d43a7ad8a612201 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/caee9bb5427744818d43a7ad8a612201 2024-11-26T19:14:15,690 DEBUG [StoreCloser-TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/ad268253061e43b88ec365621129766a to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/ad268253061e43b88ec365621129766a 2024-11-26T19:14:15,690 DEBUG [StoreCloser-TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/c42911a70e754a3eb876673cbd4e84fa to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/c42911a70e754a3eb876673cbd4e84fa 2024-11-26T19:14:15,691 DEBUG [StoreCloser-TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/84313ca47744468780bcf3b82b5296af to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/84313ca47744468780bcf3b82b5296af 2024-11-26T19:14:15,693 DEBUG [StoreCloser-TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/2d4a10652e2f46a5ac9b71d9b13c322d to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/2d4a10652e2f46a5ac9b71d9b13c322d 2024-11-26T19:14:15,694 DEBUG [StoreCloser-TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/af438f13b1204e9e8acfc13f307c89d4 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/af438f13b1204e9e8acfc13f307c89d4 2024-11-26T19:14:15,695 DEBUG [StoreCloser-TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/753c8c618f1f4d4bb636e3d4bb699e98 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/753c8c618f1f4d4bb636e3d4bb699e98 2024-11-26T19:14:15,696 DEBUG [StoreCloser-TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/a8898f8302174564861f2109f1015483 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/a8898f8302174564861f2109f1015483 2024-11-26T19:14:15,697 DEBUG [StoreCloser-TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/c4a0c5cf1f9a4ffcb4b3afd52bb65774 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/c4a0c5cf1f9a4ffcb4b3afd52bb65774 2024-11-26T19:14:15,698 DEBUG [StoreCloser-TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/8963dd6b3d994966883672717578324d to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/8963dd6b3d994966883672717578324d 2024-11-26T19:14:15,699 DEBUG [StoreCloser-TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/e340a0fc862e4e6899c9ba3cce88c90d to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/e340a0fc862e4e6899c9ba3cce88c90d 2024-11-26T19:14:15,700 DEBUG [StoreCloser-TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/7b79c14e7270430095e6008dcaeece65 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/7b79c14e7270430095e6008dcaeece65 2024-11-26T19:14:15,701 DEBUG [StoreCloser-TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/674882adcdff4f50bb75443483598cef to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/674882adcdff4f50bb75443483598cef 2024-11-26T19:14:15,702 DEBUG [StoreCloser-TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/725f293dbdb644f6b1a8146d0fcd5831 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/725f293dbdb644f6b1a8146d0fcd5831 2024-11-26T19:14:15,703 DEBUG [StoreCloser-TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/f9e4dc2be254487292a3465fa91e842d to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/f9e4dc2be254487292a3465fa91e842d 2024-11-26T19:14:15,704 DEBUG [StoreCloser-TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/13106d68607942e687f8bb2f54fa1a93 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/13106d68607942e687f8bb2f54fa1a93 2024-11-26T19:14:15,705 DEBUG [StoreCloser-TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/59f264b3fff84b77822329d7c684aa42 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/59f264b3fff84b77822329d7c684aa42 2024-11-26T19:14:15,706 DEBUG [StoreCloser-TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/8dfb3e1b0884467ba576198f7f349403 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/8dfb3e1b0884467ba576198f7f349403 2024-11-26T19:14:15,707 DEBUG [StoreCloser-TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/da15be7eff4449a1807670adea92ff11 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/da15be7eff4449a1807670adea92ff11 2024-11-26T19:14:15,708 DEBUG [StoreCloser-TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/d41bfc4dd3d348028897b897e60c3af3 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/d41bfc4dd3d348028897b897e60c3af3 2024-11-26T19:14:15,709 DEBUG [StoreCloser-TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/ff81f2d8e01346e1b7b68b1892e6f8fb to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/ff81f2d8e01346e1b7b68b1892e6f8fb 2024-11-26T19:14:15,710 DEBUG [StoreCloser-TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/73626e1537194c10bfe236f3d7188198 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/73626e1537194c10bfe236f3d7188198 2024-11-26T19:14:15,711 DEBUG [StoreCloser-TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/478b3346f9f842da835187dcdf211b46 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/478b3346f9f842da835187dcdf211b46 2024-11-26T19:14:15,712 DEBUG [StoreCloser-TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/32f6f58cb60847a5983a76e858f37cdd to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/32f6f58cb60847a5983a76e858f37cdd 2024-11-26T19:14:15,713 DEBUG [StoreCloser-TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/8bfed9f8d1dd41f392250f028c36e710 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/8bfed9f8d1dd41f392250f028c36e710 2024-11-26T19:14:15,714 DEBUG [StoreCloser-TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/7beb3a2a241847abb903b2be47e0ba8b to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/7beb3a2a241847abb903b2be47e0ba8b 2024-11-26T19:14:15,715 DEBUG [StoreCloser-TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/17034839721943708a7042d0da70a8ef to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/17034839721943708a7042d0da70a8ef 2024-11-26T19:14:15,716 DEBUG [StoreCloser-TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/4fae14e2a5174a329f8a829156d972ba to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/4fae14e2a5174a329f8a829156d972ba 2024-11-26T19:14:15,717 DEBUG [StoreCloser-TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/d5f694571dc4445dbe3751638267516d to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/d5f694571dc4445dbe3751638267516d 2024-11-26T19:14:15,718 DEBUG [StoreCloser-TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/d839b1e9763d42ef82dce75b0a8ff051 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/d839b1e9763d42ef82dce75b0a8ff051 2024-11-26T19:14:15,719 DEBUG [StoreCloser-TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/88f099863a1d4349b1948e92b9003dd8 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/88f099863a1d4349b1948e92b9003dd8 2024-11-26T19:14:15,720 DEBUG [StoreCloser-TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/cc8eef064d834d02bd014dc4d582f8bc to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/cc8eef064d834d02bd014dc4d582f8bc 2024-11-26T19:14:15,721 DEBUG [StoreCloser-TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/8212b6b252d04625a8f51140872aafcc to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/8212b6b252d04625a8f51140872aafcc 2024-11-26T19:14:15,722 DEBUG [StoreCloser-TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/2b8f3e209cb24ca38c377b95cc86782b to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/2b8f3e209cb24ca38c377b95cc86782b 2024-11-26T19:14:15,724 DEBUG [StoreCloser-TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/00b9e53947b74b6c8a3344aa957bb211 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/00b9e53947b74b6c8a3344aa957bb211 2024-11-26T19:14:15,725 DEBUG [StoreCloser-TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/4b5236a5c5d14cd6bf4d3b06b23d2f94, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/1beea36d53264c0cb8b3f76dcac2c531, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/275b2f0bc7a040b4abb39b9e69c93b2b, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/c72b1f40d9a84d0a8c0ce7fbd4efb161, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/3d0d5d31416f496e872554561169b924, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/5ccecf4f55814107ab8d667473719261, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/8a83380a02c1473689c9afa4aa69aecc, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/f1b3d656676a42feba4d88e1c212d3ba, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/966e3641f55f4a57afe91067d2a75499, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/65a9b245a4ee432d81cf41a6ab896966, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/04ac804bd0d64101b082009969bb3647, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/ca33c2a4ec0749d19d651b83f18f98f5, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/918fb8fd1f8841b2886ef357573c1940, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/4c4d3d19acd0421e9abfd9d3b9021ab0, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/e128b829b48a4a2b965d6ac11efc84d9, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/c4a004465289467ea564f3accff09f19, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/13d7bbfdd0b943cd9c869f98e44f7b4e, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/02c786736cee400e90920e425e21500b, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/fbb60340b5c64f80b794ae598ca80a8f, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/8d8b1a4596ab4d139bdbf13a5c9e37bc, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/a6683e1a65124d1bb009ce53628acb6a, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/3faf0f0527fa4971a495694e2ba1081a, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/6fab4748f5394c0eb9109fb7f84ecbaf, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/f6b18b9166df4406a886ed525bf517d5, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/9adcb6f4199242698dfb6b3e69bdac63, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/555abfe58b9a418f925cdfa01c312335, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/c7f85a14ae9649b092eb8a86f94adc8e, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/cbd4e5db7e5f4f9b9d4535ba227b9eed, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/14cbdcd5e9e24fe1bb94dfdcc261d946, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/c1a1ded1570e40b3a3b0374e98b0b035, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/c3c9b130a90342c2976b23f0821de4b4, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/77f2fd7654044f27b1014fc28c1cd7e4, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/7156b4db90524dd39986fc8a74732339, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/799a44e1cfe3493c833f2d3046f4ceb6] to archive 2024-11-26T19:14:15,726 DEBUG [StoreCloser-TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-26T19:14:15,728 DEBUG [StoreCloser-TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/4b5236a5c5d14cd6bf4d3b06b23d2f94 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/4b5236a5c5d14cd6bf4d3b06b23d2f94 2024-11-26T19:14:15,729 DEBUG [StoreCloser-TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/1beea36d53264c0cb8b3f76dcac2c531 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/1beea36d53264c0cb8b3f76dcac2c531 2024-11-26T19:14:15,730 DEBUG [StoreCloser-TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/275b2f0bc7a040b4abb39b9e69c93b2b to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/275b2f0bc7a040b4abb39b9e69c93b2b 2024-11-26T19:14:15,731 DEBUG [StoreCloser-TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/c72b1f40d9a84d0a8c0ce7fbd4efb161 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/c72b1f40d9a84d0a8c0ce7fbd4efb161 2024-11-26T19:14:15,732 DEBUG [StoreCloser-TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/3d0d5d31416f496e872554561169b924 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/3d0d5d31416f496e872554561169b924 2024-11-26T19:14:15,733 DEBUG [StoreCloser-TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/5ccecf4f55814107ab8d667473719261 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/5ccecf4f55814107ab8d667473719261 2024-11-26T19:14:15,734 DEBUG [StoreCloser-TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/8a83380a02c1473689c9afa4aa69aecc to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/8a83380a02c1473689c9afa4aa69aecc 2024-11-26T19:14:15,735 DEBUG [StoreCloser-TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/f1b3d656676a42feba4d88e1c212d3ba to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/f1b3d656676a42feba4d88e1c212d3ba 2024-11-26T19:14:15,736 DEBUG [StoreCloser-TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/966e3641f55f4a57afe91067d2a75499 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/966e3641f55f4a57afe91067d2a75499 2024-11-26T19:14:15,737 DEBUG [StoreCloser-TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/65a9b245a4ee432d81cf41a6ab896966 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/65a9b245a4ee432d81cf41a6ab896966 2024-11-26T19:14:15,738 DEBUG [StoreCloser-TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/04ac804bd0d64101b082009969bb3647 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/04ac804bd0d64101b082009969bb3647 2024-11-26T19:14:15,739 DEBUG [StoreCloser-TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/ca33c2a4ec0749d19d651b83f18f98f5 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/ca33c2a4ec0749d19d651b83f18f98f5 2024-11-26T19:14:15,740 DEBUG [StoreCloser-TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/918fb8fd1f8841b2886ef357573c1940 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/918fb8fd1f8841b2886ef357573c1940 2024-11-26T19:14:15,741 DEBUG [StoreCloser-TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/4c4d3d19acd0421e9abfd9d3b9021ab0 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/4c4d3d19acd0421e9abfd9d3b9021ab0 2024-11-26T19:14:15,742 DEBUG [StoreCloser-TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/e128b829b48a4a2b965d6ac11efc84d9 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/e128b829b48a4a2b965d6ac11efc84d9 2024-11-26T19:14:15,743 DEBUG [StoreCloser-TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/c4a004465289467ea564f3accff09f19 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/c4a004465289467ea564f3accff09f19 2024-11-26T19:14:15,744 DEBUG [StoreCloser-TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/13d7bbfdd0b943cd9c869f98e44f7b4e to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/13d7bbfdd0b943cd9c869f98e44f7b4e 2024-11-26T19:14:15,745 DEBUG [StoreCloser-TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/02c786736cee400e90920e425e21500b to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/02c786736cee400e90920e425e21500b 2024-11-26T19:14:15,746 DEBUG [StoreCloser-TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/fbb60340b5c64f80b794ae598ca80a8f to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/fbb60340b5c64f80b794ae598ca80a8f 2024-11-26T19:14:15,747 DEBUG [StoreCloser-TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/8d8b1a4596ab4d139bdbf13a5c9e37bc to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/8d8b1a4596ab4d139bdbf13a5c9e37bc 2024-11-26T19:14:15,748 DEBUG [StoreCloser-TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/a6683e1a65124d1bb009ce53628acb6a to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/a6683e1a65124d1bb009ce53628acb6a 2024-11-26T19:14:15,749 DEBUG [StoreCloser-TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/3faf0f0527fa4971a495694e2ba1081a to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/3faf0f0527fa4971a495694e2ba1081a 2024-11-26T19:14:15,750 DEBUG [StoreCloser-TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/6fab4748f5394c0eb9109fb7f84ecbaf to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/6fab4748f5394c0eb9109fb7f84ecbaf 2024-11-26T19:14:15,751 DEBUG [StoreCloser-TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/f6b18b9166df4406a886ed525bf517d5 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/f6b18b9166df4406a886ed525bf517d5 2024-11-26T19:14:15,752 DEBUG [StoreCloser-TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/9adcb6f4199242698dfb6b3e69bdac63 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/9adcb6f4199242698dfb6b3e69bdac63 2024-11-26T19:14:15,753 DEBUG [StoreCloser-TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/555abfe58b9a418f925cdfa01c312335 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/555abfe58b9a418f925cdfa01c312335 2024-11-26T19:14:15,754 DEBUG [StoreCloser-TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/c7f85a14ae9649b092eb8a86f94adc8e to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/c7f85a14ae9649b092eb8a86f94adc8e 2024-11-26T19:14:15,755 DEBUG [StoreCloser-TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/cbd4e5db7e5f4f9b9d4535ba227b9eed to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/cbd4e5db7e5f4f9b9d4535ba227b9eed 2024-11-26T19:14:15,757 DEBUG [StoreCloser-TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/14cbdcd5e9e24fe1bb94dfdcc261d946 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/14cbdcd5e9e24fe1bb94dfdcc261d946 2024-11-26T19:14:15,758 DEBUG [StoreCloser-TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/c1a1ded1570e40b3a3b0374e98b0b035 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/c1a1ded1570e40b3a3b0374e98b0b035 2024-11-26T19:14:15,759 DEBUG [StoreCloser-TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/c3c9b130a90342c2976b23f0821de4b4 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/c3c9b130a90342c2976b23f0821de4b4 2024-11-26T19:14:15,760 DEBUG [StoreCloser-TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/77f2fd7654044f27b1014fc28c1cd7e4 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/77f2fd7654044f27b1014fc28c1cd7e4 2024-11-26T19:14:15,761 DEBUG [StoreCloser-TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/7156b4db90524dd39986fc8a74732339 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/7156b4db90524dd39986fc8a74732339 2024-11-26T19:14:15,762 DEBUG [StoreCloser-TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/799a44e1cfe3493c833f2d3046f4ceb6 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/799a44e1cfe3493c833f2d3046f4ceb6 2024-11-26T19:14:15,767 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/recovered.edits/535.seqid, newMaxSeqId=535, maxSeqId=1 2024-11-26T19:14:15,768 INFO [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. 2024-11-26T19:14:15,768 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1676): Region close journal for 76c4ff0c848e5ffb5ad9989537063596: Waiting for close lock at 1732648454372Running coprocessor pre-close hooks at 1732648454372Disabling compacts and flushes for region at 1732648454372Disabling writes for close at 1732648454373 (+1 ms)Obtaining lock to block concurrent updates at 1732648454373Preparing flush snapshotting stores in 76c4ff0c848e5ffb5ad9989537063596 at 1732648454373Finished memstore snapshotting TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596., syncing WAL and waiting on mvcc, flushsize=dataSize=13740, getHeapSize=41520, getOffHeapSize=0, getCellsCount=300 at 1732648454373Flushing stores of TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596. at 1732648454374 (+1 ms)Flushing 76c4ff0c848e5ffb5ad9989537063596/A: creating writer at 1732648454374Flushing 76c4ff0c848e5ffb5ad9989537063596/A: appending metadata at 1732648454376 (+2 ms)Flushing 76c4ff0c848e5ffb5ad9989537063596/A: closing flushed file at 1732648454377 (+1 ms)Flushing 76c4ff0c848e5ffb5ad9989537063596/B: creating writer at 1732648454786 (+409 ms)Flushing 76c4ff0c848e5ffb5ad9989537063596/B: appending metadata at 1732648454789 (+3 ms)Flushing 76c4ff0c848e5ffb5ad9989537063596/B: closing flushed file at 1732648454789Flushing 76c4ff0c848e5ffb5ad9989537063596/C: creating writer at 1732648455198 (+409 ms)Flushing 76c4ff0c848e5ffb5ad9989537063596/C: appending metadata at 1732648455200 (+2 ms)Flushing 76c4ff0c848e5ffb5ad9989537063596/C: closing flushed file at 1732648455200Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7346a71e: reopening flushed file at 1732648455610 (+410 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@52b79fee: reopening flushed file at 1732648455615 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6a17c835: reopening flushed file at 1732648455619 (+4 ms)Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for 76c4ff0c848e5ffb5ad9989537063596 in 1252ms, sequenceid=532, compaction requested=true at 1732648455625 (+6 ms)Writing region close event to WAL at 1732648455764 (+139 ms)Running coprocessor post-close hooks at 1732648455768 (+4 ms)Closed at 1732648455768 2024-11-26T19:14:15,771 INFO [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] handler.UnassignRegionHandler(157): Closed 76c4ff0c848e5ffb5ad9989537063596 2024-11-26T19:14:15,771 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=76c4ff0c848e5ffb5ad9989537063596, regionState=CLOSED 2024-11-26T19:14:15,773 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=26, ppid=25, state=RUNNABLE, hasLock=false; CloseRegionProcedure 76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 because future has completed 2024-11-26T19:14:15,775 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-26T19:14:15,775 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; CloseRegionProcedure 76c4ff0c848e5ffb5ad9989537063596, server=fe76acf732b2,42785,1732648423970 in 1.5590 sec 2024-11-26T19:14:15,777 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=25, resume processing ppid=24 2024-11-26T19:14:15,777 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, ppid=24, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=76c4ff0c848e5ffb5ad9989537063596, UNASSIGN in 1.5640 sec 2024-11-26T19:14:15,780 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-26T19:14:15,780 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5710 sec 2024-11-26T19:14:15,781 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732648455781"}]},"ts":"1732648455781"} 2024-11-26T19:14:15,782 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-26T19:14:15,782 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-26T19:14:15,784 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5840 sec 2024-11-26T19:14:16,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-26T19:14:16,352 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-26T19:14:16,355 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-26T19:14:16,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] procedure2.ProcedureExecutor(1139): Stored pid=27, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T19:14:16,360 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=27, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T19:14:16,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-26T19:14:16,362 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=27, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T19:14:16,366 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596 2024-11-26T19:14:16,369 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A, FileablePath, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B, FileablePath, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C, FileablePath, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/recovered.edits] 2024-11-26T19:14:16,372 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/1b4c2ad639fa4c29b1bf71bd8d9e61b0 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/1b4c2ad639fa4c29b1bf71bd8d9e61b0 2024-11-26T19:14:16,373 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/72b52c71508a4ab3ad218a451224dfd6 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/72b52c71508a4ab3ad218a451224dfd6 2024-11-26T19:14:16,374 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/b0ab0ab0b9c649f7a3b42088c99e8dd5 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/b0ab0ab0b9c649f7a3b42088c99e8dd5 2024-11-26T19:14:16,375 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/c6371fdd17a644b3b4e96fad3e4392ed to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/A/c6371fdd17a644b3b4e96fad3e4392ed 2024-11-26T19:14:16,377 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/6cc150e9c331431f9fa9223e98ae91fb to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/6cc150e9c331431f9fa9223e98ae91fb 2024-11-26T19:14:16,379 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/91ecbfad536748c2855865b072661ba5 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/91ecbfad536748c2855865b072661ba5 2024-11-26T19:14:16,380 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/e82d431962f1464b9f1cb67e504b72a1 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/e82d431962f1464b9f1cb67e504b72a1 2024-11-26T19:14:16,381 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/eea9951c5e1045ce8384227a507535a6 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/B/eea9951c5e1045ce8384227a507535a6 2024-11-26T19:14:16,383 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/29edc9700d0a4a2696107f67660f393e to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/29edc9700d0a4a2696107f67660f393e 2024-11-26T19:14:16,385 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/3c49ff008bb24ba3924208365b34dbb9 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/3c49ff008bb24ba3924208365b34dbb9 2024-11-26T19:14:16,386 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/51bb91aa53734b9a83c429b96e91ac50 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/51bb91aa53734b9a83c429b96e91ac50 2024-11-26T19:14:16,387 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/6ea70f98984e4c2e950357dd6d639bb3 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/C/6ea70f98984e4c2e950357dd6d639bb3 2024-11-26T19:14:16,390 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/recovered.edits/535.seqid to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596/recovered.edits/535.seqid 2024-11-26T19:14:16,391 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/76c4ff0c848e5ffb5ad9989537063596 2024-11-26T19:14:16,391 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-26T19:14:16,395 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=27, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T19:14:16,399 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42785 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-26T19:14:16,404 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-26T19:14:16,438 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-26T19:14:16,439 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=27, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T19:14:16,440 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-26T19:14:16,440 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732648456440"}]},"ts":"9223372036854775807"} 2024-11-26T19:14:16,445 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-26T19:14:16,445 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 76c4ff0c848e5ffb5ad9989537063596, NAME => 'TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596.', STARTKEY => '', ENDKEY => ''}] 2024-11-26T19:14:16,445 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-26T19:14:16,445 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732648456445"}]},"ts":"9223372036854775807"} 2024-11-26T19:14:16,448 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-26T19:14:16,449 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=27, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T19:14:16,450 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 94 msec 2024-11-26T19:14:16,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-26T19:14:16,472 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-26T19:14:16,473 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-26T19:14:16,482 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=248 (was 211) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;fe76acf732b2:42785-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1331211566_22 at /127.0.0.1:48978 [Waiting for operation #362] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1480762926_22 at /127.0.0.1:46944 [Waiting for operation #349] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=453 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=448 (was 357) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=6954 (was 6774) - AvailableMemoryMB LEAK? - 2024-11-26T19:14:16,491 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=248, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=448, ProcessCount=11, AvailableMemoryMB=6953 2024-11-26T19:14:16,493 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-26T19:14:16,493 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-26T19:14:16,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] procedure2.ProcedureExecutor(1139): Stored pid=28, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-26T19:14:16,495 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-26T19:14:16,495 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:16,496 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 28 2024-11-26T19:14:16,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-26T19:14:16,496 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-26T19:14:16,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741953_1129 (size=1061) 2024-11-26T19:14:16,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-26T19:14:16,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-26T19:14:16,905 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => d23b79ada6d885c09c429afcb7484518, NAME => 'TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16 2024-11-26T19:14:16,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741954_1130 (size=53) 2024-11-26T19:14:17,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-26T19:14:17,312 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T19:14:17,312 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing d23b79ada6d885c09c429afcb7484518, disabling compactions & flushes 2024-11-26T19:14:17,312 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. 2024-11-26T19:14:17,312 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. 2024-11-26T19:14:17,312 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. after waiting 0 ms 2024-11-26T19:14:17,312 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. 2024-11-26T19:14:17,312 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. 2024-11-26T19:14:17,312 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for d23b79ada6d885c09c429afcb7484518: Waiting for close lock at 1732648457312Disabling compacts and flushes for region at 1732648457312Disabling writes for close at 1732648457312Writing region close event to WAL at 1732648457312Closed at 1732648457312 2024-11-26T19:14:17,313 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-26T19:14:17,313 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732648457313"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732648457313"}]},"ts":"1732648457313"} 2024-11-26T19:14:17,315 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-26T19:14:17,316 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-26T19:14:17,317 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732648457316"}]},"ts":"1732648457316"} 2024-11-26T19:14:17,318 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-26T19:14:17,318 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d23b79ada6d885c09c429afcb7484518, ASSIGN}] 2024-11-26T19:14:17,319 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d23b79ada6d885c09c429afcb7484518, ASSIGN 2024-11-26T19:14:17,320 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d23b79ada6d885c09c429afcb7484518, ASSIGN; state=OFFLINE, location=fe76acf732b2,42785,1732648423970; forceNewPlan=false, retain=false 2024-11-26T19:14:17,471 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=d23b79ada6d885c09c429afcb7484518, regionState=OPENING, regionLocation=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:17,473 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d23b79ada6d885c09c429afcb7484518, ASSIGN because future has completed 2024-11-26T19:14:17,473 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970}] 2024-11-26T19:14:17,628 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. 2024-11-26T19:14:17,629 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(7752): Opening region: {ENCODED => d23b79ada6d885c09c429afcb7484518, NAME => 'TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.', STARTKEY => '', ENDKEY => ''} 2024-11-26T19:14:17,629 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees d23b79ada6d885c09c429afcb7484518 2024-11-26T19:14:17,629 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T19:14:17,629 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(7794): checking encryption for d23b79ada6d885c09c429afcb7484518 2024-11-26T19:14:17,629 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(7797): checking classloading for d23b79ada6d885c09c429afcb7484518 2024-11-26T19:14:17,630 INFO [StoreOpener-d23b79ada6d885c09c429afcb7484518-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region d23b79ada6d885c09c429afcb7484518 2024-11-26T19:14:17,632 INFO [StoreOpener-d23b79ada6d885c09c429afcb7484518-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-26T19:14:17,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-26T19:14:17,632 INFO [StoreOpener-d23b79ada6d885c09c429afcb7484518-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d23b79ada6d885c09c429afcb7484518 columnFamilyName A 2024-11-26T19:14:17,633 DEBUG [StoreOpener-d23b79ada6d885c09c429afcb7484518-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:17,633 INFO [StoreOpener-d23b79ada6d885c09c429afcb7484518-1 {}] regionserver.HStore(327): Store=d23b79ada6d885c09c429afcb7484518/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T19:14:17,633 INFO [StoreOpener-d23b79ada6d885c09c429afcb7484518-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region d23b79ada6d885c09c429afcb7484518 2024-11-26T19:14:17,634 INFO [StoreOpener-d23b79ada6d885c09c429afcb7484518-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-26T19:14:17,634 INFO [StoreOpener-d23b79ada6d885c09c429afcb7484518-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d23b79ada6d885c09c429afcb7484518 columnFamilyName B 2024-11-26T19:14:17,634 DEBUG [StoreOpener-d23b79ada6d885c09c429afcb7484518-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:17,635 INFO [StoreOpener-d23b79ada6d885c09c429afcb7484518-1 {}] regionserver.HStore(327): Store=d23b79ada6d885c09c429afcb7484518/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T19:14:17,635 INFO [StoreOpener-d23b79ada6d885c09c429afcb7484518-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region d23b79ada6d885c09c429afcb7484518 2024-11-26T19:14:17,636 INFO [StoreOpener-d23b79ada6d885c09c429afcb7484518-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-26T19:14:17,636 INFO [StoreOpener-d23b79ada6d885c09c429afcb7484518-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d23b79ada6d885c09c429afcb7484518 columnFamilyName C 2024-11-26T19:14:17,636 DEBUG [StoreOpener-d23b79ada6d885c09c429afcb7484518-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:17,636 INFO [StoreOpener-d23b79ada6d885c09c429afcb7484518-1 {}] regionserver.HStore(327): Store=d23b79ada6d885c09c429afcb7484518/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T19:14:17,636 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. 2024-11-26T19:14:17,636 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1038): replaying wal for d23b79ada6d885c09c429afcb7484518 2024-11-26T19:14:17,637 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518 2024-11-26T19:14:17,637 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518 2024-11-26T19:14:17,638 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1048): stopping wal replay for d23b79ada6d885c09c429afcb7484518 2024-11-26T19:14:17,638 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1060): Cleaning up temporary data for d23b79ada6d885c09c429afcb7484518 2024-11-26T19:14:17,639 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-26T19:14:17,640 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1093): writing seq id for d23b79ada6d885c09c429afcb7484518 2024-11-26T19:14:17,642 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T19:14:17,642 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1114): Opened d23b79ada6d885c09c429afcb7484518; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67744597, jitterRate=0.009473159909248352}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-26T19:14:17,642 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d23b79ada6d885c09c429afcb7484518 2024-11-26T19:14:17,643 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1006): Region open journal for d23b79ada6d885c09c429afcb7484518: Running coprocessor pre-open hook at 1732648457629Writing region info on filesystem at 1732648457629Initializing all the Stores at 1732648457630 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732648457630Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732648457630Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732648457630Cleaning up temporary data from old regions at 1732648457638 (+8 ms)Running coprocessor post-open hooks at 1732648457642 (+4 ms)Region opened successfully at 1732648457643 (+1 ms) 2024-11-26T19:14:17,644 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., pid=30, masterSystemTime=1732648457625 2024-11-26T19:14:17,645 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. 2024-11-26T19:14:17,645 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. 2024-11-26T19:14:17,646 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=d23b79ada6d885c09c429afcb7484518, regionState=OPEN, openSeqNum=2, regionLocation=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:17,647 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 because future has completed 2024-11-26T19:14:17,650 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-26T19:14:17,650 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; OpenRegionProcedure d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 in 175 msec 2024-11-26T19:14:17,652 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=29, resume processing ppid=28 2024-11-26T19:14:17,652 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=28, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d23b79ada6d885c09c429afcb7484518, ASSIGN in 332 msec 2024-11-26T19:14:17,652 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-26T19:14:17,652 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732648457652"}]},"ts":"1732648457652"} 2024-11-26T19:14:17,654 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-26T19:14:17,654 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-26T19:14:17,655 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1610 sec 2024-11-26T19:14:18,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-26T19:14:18,642 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-26T19:14:18,643 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f46cc3c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:14:18,643 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fe76acf732b2,33811,-1 for getting cluster id 2024-11-26T19:14:18,643 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T19:14:18,645 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3cec4e24-71db-4cd0-b63a-4ea30a9b160b' 2024-11-26T19:14:18,645 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T19:14:18,645 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3cec4e24-71db-4cd0-b63a-4ea30a9b160b" 2024-11-26T19:14:18,645 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@150e67fa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:14:18,645 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fe76acf732b2,33811,-1] 2024-11-26T19:14:18,646 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T19:14:18,646 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:14:18,647 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36922, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T19:14:18,648 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ccf8029, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:14:18,648 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T19:14:18,650 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fe76acf732b2,42785,1732648423970, seqNum=-1] 2024-11-26T19:14:18,650 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T19:14:18,651 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39286, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T19:14:18,655 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-26T19:14:18,656 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is fe76acf732b2,33811,1732648423099 2024-11-26T19:14:18,656 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@227cf768 2024-11-26T19:14:18,656 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-26T19:14:18,657 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36936, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-26T19:14:18,662 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-26T19:14:18,662 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-26T19:14:18,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] procedure2.ProcedureExecutor(1139): Stored pid=31, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-26T19:14:18,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741955_1131 (size=1097) 2024-11-26T19:14:19,083 DEBUG [PEWorker-5 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-26T19:14:19,083 INFO [PEWorker-5 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-26T19:14:19,086 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-26T19:14:19,093 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d23b79ada6d885c09c429afcb7484518, REOPEN/MOVE}] 2024-11-26T19:14:19,093 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d23b79ada6d885c09c429afcb7484518, REOPEN/MOVE 2024-11-26T19:14:19,094 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=d23b79ada6d885c09c429afcb7484518, regionState=CLOSING, regionLocation=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:19,095 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d23b79ada6d885c09c429afcb7484518, REOPEN/MOVE because future has completed 2024-11-26T19:14:19,096 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-26T19:14:19,096 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE, hasLock=false; CloseRegionProcedure d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970}] 2024-11-26T19:14:19,248 INFO [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] handler.UnassignRegionHandler(122): Close d23b79ada6d885c09c429afcb7484518 2024-11-26T19:14:19,248 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-26T19:14:19,248 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1722): Closing d23b79ada6d885c09c429afcb7484518, disabling compactions & flushes 2024-11-26T19:14:19,248 INFO [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. 2024-11-26T19:14:19,249 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. 2024-11-26T19:14:19,249 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. after waiting 0 ms 2024-11-26T19:14:19,249 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. 2024-11-26T19:14:19,252 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-26T19:14:19,253 INFO [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. 2024-11-26T19:14:19,253 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1676): Region close journal for d23b79ada6d885c09c429afcb7484518: Waiting for close lock at 1732648459248Running coprocessor pre-close hooks at 1732648459248Disabling compacts and flushes for region at 1732648459248Disabling writes for close at 1732648459249 (+1 ms)Writing region close event to WAL at 1732648459249Running coprocessor post-close hooks at 1732648459253 (+4 ms)Closed at 1732648459253 2024-11-26T19:14:19,253 WARN [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegionServer(3299): Not adding moved region record: d23b79ada6d885c09c429afcb7484518 to self. 2024-11-26T19:14:19,255 INFO [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] handler.UnassignRegionHandler(157): Closed d23b79ada6d885c09c429afcb7484518 2024-11-26T19:14:19,256 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=d23b79ada6d885c09c429afcb7484518, regionState=CLOSED 2024-11-26T19:14:19,257 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=33, state=RUNNABLE, hasLock=false; CloseRegionProcedure d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 because future has completed 2024-11-26T19:14:19,260 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=33 2024-11-26T19:14:19,260 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 in 162 msec 2024-11-26T19:14:19,261 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d23b79ada6d885c09c429afcb7484518, REOPEN/MOVE; state=CLOSED, location=fe76acf732b2,42785,1732648423970; forceNewPlan=false, retain=true 2024-11-26T19:14:19,411 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=d23b79ada6d885c09c429afcb7484518, regionState=OPENING, regionLocation=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:19,413 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d23b79ada6d885c09c429afcb7484518, REOPEN/MOVE because future has completed 2024-11-26T19:14:19,414 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970}] 2024-11-26T19:14:19,569 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. 2024-11-26T19:14:19,569 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7752): Opening region: {ENCODED => d23b79ada6d885c09c429afcb7484518, NAME => 'TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.', STARTKEY => '', ENDKEY => ''} 2024-11-26T19:14:19,570 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees d23b79ada6d885c09c429afcb7484518 2024-11-26T19:14:19,570 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T19:14:19,570 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7794): checking encryption for d23b79ada6d885c09c429afcb7484518 2024-11-26T19:14:19,570 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7797): checking classloading for d23b79ada6d885c09c429afcb7484518 2024-11-26T19:14:19,572 INFO [StoreOpener-d23b79ada6d885c09c429afcb7484518-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region d23b79ada6d885c09c429afcb7484518 2024-11-26T19:14:19,573 INFO [StoreOpener-d23b79ada6d885c09c429afcb7484518-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-26T19:14:19,579 INFO [StoreOpener-d23b79ada6d885c09c429afcb7484518-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d23b79ada6d885c09c429afcb7484518 columnFamilyName A 2024-11-26T19:14:19,581 DEBUG [StoreOpener-d23b79ada6d885c09c429afcb7484518-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:19,581 INFO [StoreOpener-d23b79ada6d885c09c429afcb7484518-1 {}] regionserver.HStore(327): Store=d23b79ada6d885c09c429afcb7484518/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T19:14:19,582 INFO [StoreOpener-d23b79ada6d885c09c429afcb7484518-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region d23b79ada6d885c09c429afcb7484518 2024-11-26T19:14:19,583 INFO [StoreOpener-d23b79ada6d885c09c429afcb7484518-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-26T19:14:19,583 INFO [StoreOpener-d23b79ada6d885c09c429afcb7484518-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d23b79ada6d885c09c429afcb7484518 columnFamilyName B 2024-11-26T19:14:19,583 DEBUG [StoreOpener-d23b79ada6d885c09c429afcb7484518-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:19,583 INFO [StoreOpener-d23b79ada6d885c09c429afcb7484518-1 {}] regionserver.HStore(327): Store=d23b79ada6d885c09c429afcb7484518/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T19:14:19,584 INFO [StoreOpener-d23b79ada6d885c09c429afcb7484518-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region d23b79ada6d885c09c429afcb7484518 2024-11-26T19:14:19,584 INFO [StoreOpener-d23b79ada6d885c09c429afcb7484518-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-26T19:14:19,584 INFO [StoreOpener-d23b79ada6d885c09c429afcb7484518-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d23b79ada6d885c09c429afcb7484518 columnFamilyName C 2024-11-26T19:14:19,584 DEBUG [StoreOpener-d23b79ada6d885c09c429afcb7484518-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:19,585 INFO [StoreOpener-d23b79ada6d885c09c429afcb7484518-1 {}] regionserver.HStore(327): Store=d23b79ada6d885c09c429afcb7484518/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T19:14:19,585 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. 2024-11-26T19:14:19,585 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1038): replaying wal for d23b79ada6d885c09c429afcb7484518 2024-11-26T19:14:19,586 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518 2024-11-26T19:14:19,587 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518 2024-11-26T19:14:19,587 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1048): stopping wal replay for d23b79ada6d885c09c429afcb7484518 2024-11-26T19:14:19,587 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1060): Cleaning up temporary data for d23b79ada6d885c09c429afcb7484518 2024-11-26T19:14:19,588 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-26T19:14:19,589 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1093): writing seq id for d23b79ada6d885c09c429afcb7484518 2024-11-26T19:14:19,590 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1114): Opened d23b79ada6d885c09c429afcb7484518; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69430141, jitterRate=0.034589722752571106}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-26T19:14:19,590 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d23b79ada6d885c09c429afcb7484518 2024-11-26T19:14:19,591 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1006): Region open journal for d23b79ada6d885c09c429afcb7484518: Running coprocessor pre-open hook at 1732648459570Writing region info on filesystem at 1732648459570Initializing all the Stores at 1732648459571 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732648459571Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732648459571Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732648459571Cleaning up temporary data from old regions at 1732648459587 (+16 ms)Running coprocessor post-open hooks at 1732648459590 (+3 ms)Region opened successfully at 1732648459591 (+1 ms) 2024-11-26T19:14:19,592 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., pid=35, masterSystemTime=1732648459566 2024-11-26T19:14:19,594 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. 2024-11-26T19:14:19,594 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. 2024-11-26T19:14:19,595 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=d23b79ada6d885c09c429afcb7484518, regionState=OPEN, openSeqNum=5, regionLocation=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:19,596 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 because future has completed 2024-11-26T19:14:19,599 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=33 2024-11-26T19:14:19,599 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=33, state=SUCCESS, hasLock=false; OpenRegionProcedure d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 in 184 msec 2024-11-26T19:14:19,600 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-11-26T19:14:19,600 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d23b79ada6d885c09c429afcb7484518, REOPEN/MOVE in 506 msec 2024-11-26T19:14:19,603 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-11-26T19:14:19,603 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 515 msec 2024-11-26T19:14:19,606 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 939 msec 2024-11-26T19:14:19,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-26T19:14:19,608 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-26T19:14:19,608 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@88c9976, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:14:19,608 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fe76acf732b2,33811,-1 for getting cluster id 2024-11-26T19:14:19,609 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T19:14:19,610 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3cec4e24-71db-4cd0-b63a-4ea30a9b160b' 2024-11-26T19:14:19,610 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T19:14:19,610 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3cec4e24-71db-4cd0-b63a-4ea30a9b160b" 2024-11-26T19:14:19,610 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@192b075, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:14:19,610 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fe76acf732b2,33811,-1] 2024-11-26T19:14:19,610 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T19:14:19,610 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:14:19,611 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36952, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T19:14:19,612 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23b42a95, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:14:19,613 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7aa768d0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:14:19,613 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fe76acf732b2,33811,-1 for getting cluster id 2024-11-26T19:14:19,613 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T19:14:19,614 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3cec4e24-71db-4cd0-b63a-4ea30a9b160b' 2024-11-26T19:14:19,614 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T19:14:19,614 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3cec4e24-71db-4cd0-b63a-4ea30a9b160b" 2024-11-26T19:14:19,614 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51dfab01, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:14:19,614 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fe76acf732b2,33811,-1] 2024-11-26T19:14:19,614 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T19:14:19,614 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:14:19,615 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36976, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T19:14:19,615 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@318b0247, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:14:19,617 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2bc6bab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:14:19,617 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fe76acf732b2,33811,-1 for getting cluster id 2024-11-26T19:14:19,617 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T19:14:19,618 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3cec4e24-71db-4cd0-b63a-4ea30a9b160b' 2024-11-26T19:14:19,618 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T19:14:19,618 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3cec4e24-71db-4cd0-b63a-4ea30a9b160b" 2024-11-26T19:14:19,618 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d738fe4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:14:19,618 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fe76acf732b2,33811,-1] 2024-11-26T19:14:19,618 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T19:14:19,618 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:14:19,619 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36990, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T19:14:19,619 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79d8e0bb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:14:19,620 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ddd470f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:14:19,620 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fe76acf732b2,33811,-1 for getting cluster id 2024-11-26T19:14:19,620 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T19:14:19,621 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3cec4e24-71db-4cd0-b63a-4ea30a9b160b' 2024-11-26T19:14:19,621 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T19:14:19,621 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3cec4e24-71db-4cd0-b63a-4ea30a9b160b" 2024-11-26T19:14:19,621 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@76129c9a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:14:19,621 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fe76acf732b2,33811,-1] 2024-11-26T19:14:19,622 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T19:14:19,622 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:14:19,622 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37014, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T19:14:19,623 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63050805, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:14:19,623 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60093174, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:14:19,623 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fe76acf732b2,33811,-1 for getting cluster id 2024-11-26T19:14:19,624 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T19:14:19,624 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3cec4e24-71db-4cd0-b63a-4ea30a9b160b' 2024-11-26T19:14:19,624 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T19:14:19,624 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3cec4e24-71db-4cd0-b63a-4ea30a9b160b" 2024-11-26T19:14:19,624 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b60be31, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:14:19,624 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fe76acf732b2,33811,-1] 2024-11-26T19:14:19,625 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T19:14:19,625 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:14:19,625 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37032, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T19:14:19,626 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a670096, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:14:19,627 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f8203f7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:14:19,627 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fe76acf732b2,33811,-1 for getting cluster id 2024-11-26T19:14:19,627 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T19:14:19,627 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3cec4e24-71db-4cd0-b63a-4ea30a9b160b' 2024-11-26T19:14:19,627 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T19:14:19,628 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3cec4e24-71db-4cd0-b63a-4ea30a9b160b" 2024-11-26T19:14:19,628 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a8caea3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:14:19,628 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fe76acf732b2,33811,-1] 2024-11-26T19:14:19,628 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T19:14:19,628 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:14:19,629 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37048, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T19:14:19,629 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d7c6386, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:14:19,630 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6723353c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:14:19,630 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fe76acf732b2,33811,-1 for getting cluster id 2024-11-26T19:14:19,630 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T19:14:19,630 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3cec4e24-71db-4cd0-b63a-4ea30a9b160b' 2024-11-26T19:14:19,631 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T19:14:19,631 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3cec4e24-71db-4cd0-b63a-4ea30a9b160b" 2024-11-26T19:14:19,631 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7dfc90db, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:14:19,631 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fe76acf732b2,33811,-1] 2024-11-26T19:14:19,631 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T19:14:19,631 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:14:19,632 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37072, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T19:14:19,632 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@771a0b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:14:19,633 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@aece006, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:14:19,633 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fe76acf732b2,33811,-1 for getting cluster id 2024-11-26T19:14:19,633 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T19:14:19,633 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3cec4e24-71db-4cd0-b63a-4ea30a9b160b' 2024-11-26T19:14:19,633 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T19:14:19,633 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3cec4e24-71db-4cd0-b63a-4ea30a9b160b" 2024-11-26T19:14:19,634 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70c7e2da, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:14:19,634 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fe76acf732b2,33811,-1] 2024-11-26T19:14:19,634 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T19:14:19,634 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:14:19,634 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37084, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T19:14:19,635 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22644b0d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:14:19,636 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@fe9245f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:14:19,636 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fe76acf732b2,33811,-1 for getting cluster id 2024-11-26T19:14:19,636 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T19:14:19,636 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3cec4e24-71db-4cd0-b63a-4ea30a9b160b' 2024-11-26T19:14:19,636 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T19:14:19,637 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3cec4e24-71db-4cd0-b63a-4ea30a9b160b" 2024-11-26T19:14:19,637 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@441c7907, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:14:19,637 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fe76acf732b2,33811,-1] 2024-11-26T19:14:19,637 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T19:14:19,637 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:14:19,638 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37108, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T19:14:19,638 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12988e5d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:14:19,639 DEBUG [Thread-658 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T19:14:19,639 DEBUG [Thread-669 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T19:14:19,639 DEBUG [Thread-654 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T19:14:19,639 DEBUG [Thread-652 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T19:14:19,639 DEBUG [Thread-665 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T19:14:19,639 DEBUG [Thread-660 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T19:14:19,639 DEBUG [Thread-656 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T19:14:19,640 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fe76acf732b2,42785,1732648423970, seqNum=-1] 2024-11-26T19:14:19,641 DEBUG [Thread-658 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fe76acf732b2,42785,1732648423970, seqNum=-1] 2024-11-26T19:14:19,641 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fe76acf732b2,42785,1732648423970, seqNum=-1] 2024-11-26T19:14:19,641 DEBUG [Thread-663 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T19:14:19,641 DEBUG [Thread-667 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T19:14:19,641 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fe76acf732b2,42785,1732648423970, seqNum=-1] 2024-11-26T19:14:19,642 DEBUG [Thread-658 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T19:14:19,642 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T19:14:19,642 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fe76acf732b2,42785,1732648423970, seqNum=-1] 2024-11-26T19:14:19,642 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T19:14:19,642 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T19:14:19,642 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fe76acf732b2,42785,1732648423970, seqNum=-1] 2024-11-26T19:14:19,642 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T19:14:19,643 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39292, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T19:14:19,643 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39316, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T19:14:19,643 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T19:14:19,643 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fe76acf732b2,42785,1732648423970, seqNum=-1] 2024-11-26T19:14:19,643 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39324, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T19:14:19,643 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fe76acf732b2,42785,1732648423970, seqNum=-1] 2024-11-26T19:14:19,644 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T19:14:19,644 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T19:14:19,644 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fe76acf732b2,42785,1732648423970, seqNum=-1] 2024-11-26T19:14:19,644 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39306, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T19:14:19,644 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T19:14:19,644 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39332, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T19:14:19,645 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39344, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T19:14:19,645 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39350, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T19:14:19,645 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39360, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T19:14:19,645 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39368, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T19:14:19,647 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T19:14:19,647 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5] 2024-11-26T19:14:19,647 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5] 2024-11-26T19:14:19,649 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5] 2024-11-26T19:14:19,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] procedure2.ProcedureExecutor(1139): Stored pid=36, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees 2024-11-26T19:14:19,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-26T19:14:19,651 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5] 2024-11-26T19:14:19,651 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5] 2024-11-26T19:14:19,651 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=36, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T19:14:19,653 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5] 2024-11-26T19:14:19,653 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=36, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T19:14:19,653 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T19:14:19,654 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5] 2024-11-26T19:14:19,654 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5] 2024-11-26T19:14:19,654 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5] 2024-11-26T19:14:19,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(8855): Flush requested on d23b79ada6d885c09c429afcb7484518 2024-11-26T19:14:19,671 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d23b79ada6d885c09c429afcb7484518 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-26T19:14:19,673 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d23b79ada6d885c09c429afcb7484518, store=A 2024-11-26T19:14:19,673 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:14:19,673 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d23b79ada6d885c09c429afcb7484518, store=B 2024-11-26T19:14:19,673 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:14:19,673 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d23b79ada6d885c09c429afcb7484518, store=C 2024-11-26T19:14:19,673 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:14:19,714 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:19,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39350 deadline: 1732648519702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:19,716 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:19,716 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:19,716 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:19,716 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:19,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39306 deadline: 1732648519704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:19,717 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:19,717 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:19,717 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:19,717 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:19,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39324 deadline: 1732648519707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:19,719 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:19,719 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:19,719 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:19,722 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732648459691, max timestamp=1732648459711Num uniques 859; 2024-11-26T19:14:19,723 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-26T19:14:19,724 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732648459691, max timestamp=1732648459711Num uniques 859; 2024-11-26T19:14:19,724 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-26T19:14:19,725 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732648459691, max timestamp=1732648459711Num uniques 859; 2024-11-26T19:14:19,725 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-26T19:14:19,735 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126270dd33e37ee46e8b906d207d2b1c347_d23b79ada6d885c09c429afcb7484518 is 50, key is test_row_0/A:col10/1732648459656/Put/seqid=0 2024-11-26T19:14:19,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-26T19:14:19,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741956_1132 (size=12154) 2024-11-26T19:14:19,763 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:19,770 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126270dd33e37ee46e8b906d207d2b1c347_d23b79ada6d885c09c429afcb7484518 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126270dd33e37ee46e8b906d207d2b1c347_d23b79ada6d885c09c429afcb7484518 2024-11-26T19:14:19,771 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/A/da470e667c0d413c9473b3bd29f894ac, store: [table=TestAcidGuarantees family=A region=d23b79ada6d885c09c429afcb7484518] 2024-11-26T19:14:19,781 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/A/da470e667c0d413c9473b3bd29f894ac is 175, key is test_row_0/A:col10/1732648459656/Put/seqid=0 2024-11-26T19:14:19,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741957_1133 (size=30955) 2024-11-26T19:14:19,799 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:19,799 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/A/da470e667c0d413c9473b3bd29f894ac 2024-11-26T19:14:19,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39292 deadline: 1732648519791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:19,800 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:19,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39332 deadline: 1732648519793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:19,800 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:19,800 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:19,800 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:19,800 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:19,801 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:19,801 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:19,807 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-26T19:14:19,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. 2024-11-26T19:14:19,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. as already flushing 2024-11-26T19:14:19,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. 2024-11-26T19:14:19,808 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => d23b79ada6d885c09c429afcb7484518, NAME => 'TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:19,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => d23b79ada6d885c09c429afcb7484518, NAME => 'TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:19,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d23b79ada6d885c09c429afcb7484518, NAME => 'TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d23b79ada6d885c09c429afcb7484518, NAME => 'TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:19,823 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:19,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39306 deadline: 1732648519822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:19,824 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:19,824 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:19,824 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:19,825 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:19,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39350 deadline: 1732648519822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:19,826 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:19,826 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:19,826 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:19,826 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:19,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39324 deadline: 1732648519822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:19,827 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:19,827 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:19,827 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:19,836 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/B/ad8bcceb72fe4a1a8b85bc1fcab0ebf7 is 50, key is test_row_0/B:col10/1732648459656/Put/seqid=0 2024-11-26T19:14:19,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741958_1134 (size=12001) 2024-11-26T19:14:19,847 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/B/ad8bcceb72fe4a1a8b85bc1fcab0ebf7 2024-11-26T19:14:19,886 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/C/8f94eb4760c445b3825fc75efb49f2e0 is 50, key is test_row_0/C:col10/1732648459656/Put/seqid=0 2024-11-26T19:14:19,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741959_1135 (size=12001) 2024-11-26T19:14:19,903 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/C/8f94eb4760c445b3825fc75efb49f2e0 2024-11-26T19:14:19,904 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:19,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39292 deadline: 1732648519902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:19,904 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:19,904 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:19,904 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:19,910 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/A/da470e667c0d413c9473b3bd29f894ac as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/da470e667c0d413c9473b3bd29f894ac 2024-11-26T19:14:19,913 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:19,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39332 deadline: 1732648519912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:19,914 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:19,914 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:19,914 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:19,918 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/da470e667c0d413c9473b3bd29f894ac, entries=150, sequenceid=15, filesize=30.2 K 2024-11-26T19:14:19,920 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/B/ad8bcceb72fe4a1a8b85bc1fcab0ebf7 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/ad8bcceb72fe4a1a8b85bc1fcab0ebf7 2024-11-26T19:14:19,925 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/ad8bcceb72fe4a1a8b85bc1fcab0ebf7, entries=150, sequenceid=15, filesize=11.7 K 2024-11-26T19:14:19,927 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/C/8f94eb4760c445b3825fc75efb49f2e0 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/8f94eb4760c445b3825fc75efb49f2e0 2024-11-26T19:14:19,932 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/8f94eb4760c445b3825fc75efb49f2e0, entries=150, sequenceid=15, filesize=11.7 K 2024-11-26T19:14:19,933 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for d23b79ada6d885c09c429afcb7484518 in 262ms, sequenceid=15, compaction requested=false 2024-11-26T19:14:19,933 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-26T19:14:19,934 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d23b79ada6d885c09c429afcb7484518: 2024-11-26T19:14:19,962 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-26T19:14:19,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. 2024-11-26T19:14:19,963 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2902): Flushing d23b79ada6d885c09c429afcb7484518 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-11-26T19:14:19,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d23b79ada6d885c09c429afcb7484518, store=A 2024-11-26T19:14:19,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T19:14:19,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d23b79ada6d885c09c429afcb7484518, store=B 2024-11-26T19:14:19,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T19:14:19,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d23b79ada6d885c09c429afcb7484518, store=C 2024-11-26T19:14:19,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T19:14:19,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-26T19:14:19,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126fd41d313179546078f060d1c132ba1bc_d23b79ada6d885c09c429afcb7484518 is 50, key is test_row_0/A:col10/1732648459790/Put/seqid=0 2024-11-26T19:14:20,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741960_1136 (size=12154) 2024-11-26T19:14:20,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:20,017 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126fd41d313179546078f060d1c132ba1bc_d23b79ada6d885c09c429afcb7484518 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126fd41d313179546078f060d1c132ba1bc_d23b79ada6d885c09c429afcb7484518 2024-11-26T19:14:20,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/A/d96b98aca1e442a1b92272f6a117f7e5, store: [table=TestAcidGuarantees family=A region=d23b79ada6d885c09c429afcb7484518] 2024-11-26T19:14:20,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/A/d96b98aca1e442a1b92272f6a117f7e5 is 175, key is test_row_0/A:col10/1732648459790/Put/seqid=0 2024-11-26T19:14:20,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(8855): Flush requested on d23b79ada6d885c09c429afcb7484518 2024-11-26T19:14:20,036 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. as already flushing 2024-11-26T19:14:20,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741961_1137 (size=30955) 2024-11-26T19:14:20,045 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=50, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/A/d96b98aca1e442a1b92272f6a117f7e5 2024-11-26T19:14:20,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/B/38dbac8e3b294b3da557f07d311f730d is 50, key is test_row_0/B:col10/1732648459790/Put/seqid=0 2024-11-26T19:14:20,060 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:20,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39324 deadline: 1732648520052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:20,061 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:20,061 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:20,061 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:20,064 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:20,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39350 deadline: 1732648520057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:20,065 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:20,065 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:20,065 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:20,065 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:20,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39306 deadline: 1732648520059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:20,066 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:20,066 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:20,066 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:20,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741962_1138 (size=12001) 2024-11-26T19:14:20,115 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:20,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39292 deadline: 1732648520112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:20,116 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:20,116 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:20,116 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:20,129 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:20,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39332 deadline: 1732648520122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:20,131 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:20,132 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:20,132 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:20,176 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:20,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39324 deadline: 1732648520171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:20,177 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:20,177 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:20,177 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:20,177 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:20,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39350 deadline: 1732648520172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:20,178 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:20,178 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:20,178 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:20,178 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:20,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39306 deadline: 1732648520172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:20,178 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:20,178 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:20,178 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:20,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-26T19:14:20,383 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:20,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39324 deadline: 1732648520382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:20,384 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:20,384 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:20,384 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:20,384 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:20,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39350 deadline: 1732648520383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:20,384 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:20,384 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:20,384 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:20,385 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:20,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39306 deadline: 1732648520383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:20,385 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:20,385 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:20,385 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:20,424 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:20,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39292 deadline: 1732648520422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:20,426 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:20,426 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:20,426 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:20,447 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:20,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39332 deadline: 1732648520441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:20,448 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:20,448 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:20,448 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:20,471 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/B/38dbac8e3b294b3da557f07d311f730d 2024-11-26T19:14:20,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/C/9dad166e46f84480a73d811efbede772 is 50, key is test_row_0/C:col10/1732648459790/Put/seqid=0 2024-11-26T19:14:20,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741963_1139 (size=12001) 2024-11-26T19:14:20,503 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/C/9dad166e46f84480a73d811efbede772 2024-11-26T19:14:20,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/A/d96b98aca1e442a1b92272f6a117f7e5 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/d96b98aca1e442a1b92272f6a117f7e5 2024-11-26T19:14:20,522 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/d96b98aca1e442a1b92272f6a117f7e5, entries=150, sequenceid=50, filesize=30.2 K 2024-11-26T19:14:20,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/B/38dbac8e3b294b3da557f07d311f730d as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/38dbac8e3b294b3da557f07d311f730d 2024-11-26T19:14:20,530 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/38dbac8e3b294b3da557f07d311f730d, entries=150, sequenceid=50, filesize=11.7 K 2024-11-26T19:14:20,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/C/9dad166e46f84480a73d811efbede772 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/9dad166e46f84480a73d811efbede772 2024-11-26T19:14:20,538 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/9dad166e46f84480a73d811efbede772, entries=150, sequenceid=50, filesize=11.7 K 2024-11-26T19:14:20,539 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=53.67 KB/54960 for d23b79ada6d885c09c429afcb7484518 in 576ms, sequenceid=50, compaction requested=false 2024-11-26T19:14:20,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2603): Flush status journal for d23b79ada6d885c09c429afcb7484518: 2024-11-26T19:14:20,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. 2024-11-26T19:14:20,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=37 2024-11-26T19:14:20,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster(4169): Remote procedure done, pid=37 2024-11-26T19:14:20,544 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=36 2024-11-26T19:14:20,544 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=36, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 888 msec 2024-11-26T19:14:20,546 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees in 898 msec 2024-11-26T19:14:20,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(8855): Flush requested on d23b79ada6d885c09c429afcb7484518 2024-11-26T19:14:20,696 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d23b79ada6d885c09c429afcb7484518 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-26T19:14:20,702 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d23b79ada6d885c09c429afcb7484518, store=A 2024-11-26T19:14:20,703 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:14:20,703 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d23b79ada6d885c09c429afcb7484518, store=B 2024-11-26T19:14:20,703 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:14:20,703 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d23b79ada6d885c09c429afcb7484518, store=C 2024-11-26T19:14:20,703 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:14:20,725 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411265f50b4af36cf4beab4ed28f4233550ad_d23b79ada6d885c09c429afcb7484518 is 50, key is test_row_0/A:col10/1732648460696/Put/seqid=0 2024-11-26T19:14:20,738 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:20,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39350 deadline: 1732648520733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:20,739 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:20,739 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:20,739 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:20,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741964_1140 (size=14594) 2024-11-26T19:14:20,743 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:20,744 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:20,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39306 deadline: 1732648520737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:20,745 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:20,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39324 deadline: 1732648520739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:20,746 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:20,746 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:20,746 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:20,747 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:20,747 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:20,748 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:20,754 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411265f50b4af36cf4beab4ed28f4233550ad_d23b79ada6d885c09c429afcb7484518 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411265f50b4af36cf4beab4ed28f4233550ad_d23b79ada6d885c09c429afcb7484518 2024-11-26T19:14:20,755 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/A/ca4f75ea08b041fcabc7c158fe66b6b6, store: [table=TestAcidGuarantees family=A region=d23b79ada6d885c09c429afcb7484518] 2024-11-26T19:14:20,756 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/A/ca4f75ea08b041fcabc7c158fe66b6b6 is 175, key is test_row_0/A:col10/1732648460696/Put/seqid=0 2024-11-26T19:14:20,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741965_1141 (size=39549) 2024-11-26T19:14:20,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-26T19:14:20,792 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T19:14:20,793 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T19:14:20,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] procedure2.ProcedureExecutor(1139): Stored pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees 2024-11-26T19:14:20,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-26T19:14:20,796 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T19:14:20,797 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T19:14:20,798 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T19:14:20,844 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:20,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39350 deadline: 1732648520842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:20,847 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:20,847 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:20,847 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:20,854 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:20,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39324 deadline: 1732648520852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:20,855 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:20,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39306 deadline: 1732648520852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:20,857 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:20,857 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:20,857 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:20,857 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:20,857 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:20,857 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:20,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-26T19:14:20,948 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:20,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39292 deadline: 1732648520942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:20,949 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:20,949 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:20,949 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:20,950 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-26T19:14:20,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. 2024-11-26T19:14:20,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. as already flushing 2024-11-26T19:14:20,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. 2024-11-26T19:14:20,951 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => d23b79ada6d885c09c429afcb7484518, NAME => 'TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:20,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => d23b79ada6d885c09c429afcb7484518, NAME => 'TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:20,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d23b79ada6d885c09c429afcb7484518, NAME => 'TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d23b79ada6d885c09c429afcb7484518, NAME => 'TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:20,954 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:20,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39332 deadline: 1732648520952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:20,956 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:20,956 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:20,956 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:21,053 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:21,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39350 deadline: 1732648521052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:21,054 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:21,054 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:21,054 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:21,064 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:21,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39306 deadline: 1732648521062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:21,064 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:21,065 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:21,065 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:21,065 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:21,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39324 deadline: 1732648521062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:21,066 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:21,066 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:21,066 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:21,104 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-26T19:14:21,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. 2024-11-26T19:14:21,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. as already flushing 2024-11-26T19:14:21,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. 2024-11-26T19:14:21,105 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => d23b79ada6d885c09c429afcb7484518, NAME => 'TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:21,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => d23b79ada6d885c09c429afcb7484518, NAME => 'TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:21,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d23b79ada6d885c09c429afcb7484518, NAME => 'TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d23b79ada6d885c09c429afcb7484518, NAME => 'TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:21,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-26T19:14:21,192 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=64, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/A/ca4f75ea08b041fcabc7c158fe66b6b6 2024-11-26T19:14:21,211 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/B/3b6bf4ae7c4b4988b33edda9ed1fe305 is 50, key is test_row_0/B:col10/1732648460696/Put/seqid=0 2024-11-26T19:14:21,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741966_1142 (size=12001) 2024-11-26T19:14:21,231 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/B/3b6bf4ae7c4b4988b33edda9ed1fe305 2024-11-26T19:14:21,243 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/C/d65e62fe075341f383973364f61ba044 is 50, key is test_row_0/C:col10/1732648460696/Put/seqid=0 2024-11-26T19:14:21,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741967_1143 (size=12001) 2024-11-26T19:14:21,258 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-26T19:14:21,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. 2024-11-26T19:14:21,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. as already flushing 2024-11-26T19:14:21,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. 2024-11-26T19:14:21,259 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => d23b79ada6d885c09c429afcb7484518, NAME => 'TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:21,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => d23b79ada6d885c09c429afcb7484518, NAME => 'TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:21,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d23b79ada6d885c09c429afcb7484518, NAME => 'TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d23b79ada6d885c09c429afcb7484518, NAME => 'TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:21,322 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-26T19:14:21,365 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:21,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39350 deadline: 1732648521362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:21,366 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:21,366 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:21,366 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:21,373 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:21,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39306 deadline: 1732648521372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:21,374 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:21,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39324 deadline: 1732648521372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:21,374 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:21,374 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:21,374 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:21,375 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:21,375 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:21,375 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:21,411 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-26T19:14:21,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. 2024-11-26T19:14:21,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. as already flushing 2024-11-26T19:14:21,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. 2024-11-26T19:14:21,412 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => d23b79ada6d885c09c429afcb7484518, NAME => 'TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:21,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => d23b79ada6d885c09c429afcb7484518, NAME => 'TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:21,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d23b79ada6d885c09c429afcb7484518, NAME => 'TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d23b79ada6d885c09c429afcb7484518, NAME => 'TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:21,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-26T19:14:21,565 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-26T19:14:21,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. 2024-11-26T19:14:21,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. as already flushing 2024-11-26T19:14:21,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. 2024-11-26T19:14:21,566 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => d23b79ada6d885c09c429afcb7484518, NAME => 'TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:21,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => d23b79ada6d885c09c429afcb7484518, NAME => 'TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:21,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d23b79ada6d885c09c429afcb7484518, NAME => 'TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d23b79ada6d885c09c429afcb7484518, NAME => 'TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:21,651 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/C/d65e62fe075341f383973364f61ba044 2024-11-26T19:14:21,658 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/A/ca4f75ea08b041fcabc7c158fe66b6b6 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/ca4f75ea08b041fcabc7c158fe66b6b6 2024-11-26T19:14:21,665 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/ca4f75ea08b041fcabc7c158fe66b6b6, entries=200, sequenceid=64, filesize=38.6 K 2024-11-26T19:14:21,666 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/B/3b6bf4ae7c4b4988b33edda9ed1fe305 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/3b6bf4ae7c4b4988b33edda9ed1fe305 2024-11-26T19:14:21,672 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/3b6bf4ae7c4b4988b33edda9ed1fe305, entries=150, sequenceid=64, filesize=11.7 K 2024-11-26T19:14:21,673 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/C/d65e62fe075341f383973364f61ba044 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/d65e62fe075341f383973364f61ba044 2024-11-26T19:14:21,686 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/d65e62fe075341f383973364f61ba044, entries=150, sequenceid=64, filesize=11.7 K 2024-11-26T19:14:21,687 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for d23b79ada6d885c09c429afcb7484518 in 992ms, sequenceid=64, compaction requested=true 2024-11-26T19:14:21,687 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d23b79ada6d885c09c429afcb7484518: 2024-11-26T19:14:21,687 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d23b79ada6d885c09c429afcb7484518:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T19:14:21,687 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:14:21,687 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T19:14:21,687 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d23b79ada6d885c09c429afcb7484518:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T19:14:21,687 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:14:21,687 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T19:14:21,687 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d23b79ada6d885c09c429afcb7484518:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T19:14:21,687 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T19:14:21,688 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T19:14:21,689 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T19:14:21,689 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1541): d23b79ada6d885c09c429afcb7484518/B is initiating minor compaction (all files) 2024-11-26T19:14:21,689 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1541): d23b79ada6d885c09c429afcb7484518/A is initiating minor compaction (all files) 2024-11-26T19:14:21,689 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d23b79ada6d885c09c429afcb7484518/B in TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. 2024-11-26T19:14:21,689 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d23b79ada6d885c09c429afcb7484518/A in TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. 2024-11-26T19:14:21,689 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/ad8bcceb72fe4a1a8b85bc1fcab0ebf7, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/38dbac8e3b294b3da557f07d311f730d, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/3b6bf4ae7c4b4988b33edda9ed1fe305] into tmpdir=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp, totalSize=35.2 K 2024-11-26T19:14:21,689 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/da470e667c0d413c9473b3bd29f894ac, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/d96b98aca1e442a1b92272f6a117f7e5, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/ca4f75ea08b041fcabc7c158fe66b6b6] into tmpdir=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp, totalSize=99.1 K 2024-11-26T19:14:21,689 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting ad8bcceb72fe4a1a8b85bc1fcab0ebf7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732648459656 2024-11-26T19:14:21,689 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. 2024-11-26T19:14:21,689 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. files: [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/da470e667c0d413c9473b3bd29f894ac, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/d96b98aca1e442a1b92272f6a117f7e5, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/ca4f75ea08b041fcabc7c158fe66b6b6] 2024-11-26T19:14:21,690 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting 38dbac8e3b294b3da557f07d311f730d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732648459751 2024-11-26T19:14:21,690 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting da470e667c0d413c9473b3bd29f894ac, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732648459656 2024-11-26T19:14:21,691 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting d96b98aca1e442a1b92272f6a117f7e5, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732648459751 2024-11-26T19:14:21,691 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting 3b6bf4ae7c4b4988b33edda9ed1fe305, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1732648460056 2024-11-26T19:14:21,691 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting ca4f75ea08b041fcabc7c158fe66b6b6, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1732648460056 2024-11-26T19:14:21,702 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d23b79ada6d885c09c429afcb7484518#B#compaction#125 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T19:14:21,703 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/B/4296d4424a504e21a4a50ce8beba0baa is 50, key is test_row_0/B:col10/1732648460696/Put/seqid=0 2024-11-26T19:14:21,706 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d23b79ada6d885c09c429afcb7484518] 2024-11-26T19:14:21,711 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241126fc76e93db6c7463788078f4f2a16b67a_d23b79ada6d885c09c429afcb7484518 store=[table=TestAcidGuarantees family=A region=d23b79ada6d885c09c429afcb7484518] 2024-11-26T19:14:21,716 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241126fc76e93db6c7463788078f4f2a16b67a_d23b79ada6d885c09c429afcb7484518, store=[table=TestAcidGuarantees family=A region=d23b79ada6d885c09c429afcb7484518] 2024-11-26T19:14:21,716 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126fc76e93db6c7463788078f4f2a16b67a_d23b79ada6d885c09c429afcb7484518 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d23b79ada6d885c09c429afcb7484518] 2024-11-26T19:14:21,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741968_1144 (size=12104) 2024-11-26T19:14:21,720 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-26T19:14:21,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. 2024-11-26T19:14:21,720 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2902): Flushing d23b79ada6d885c09c429afcb7484518 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-26T19:14:21,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d23b79ada6d885c09c429afcb7484518, store=A 2024-11-26T19:14:21,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:14:21,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d23b79ada6d885c09c429afcb7484518, store=B 2024-11-26T19:14:21,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:14:21,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d23b79ada6d885c09c429afcb7484518, store=C 2024-11-26T19:14:21,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:14:21,737 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/B/4296d4424a504e21a4a50ce8beba0baa as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/4296d4424a504e21a4a50ce8beba0baa 2024-11-26T19:14:21,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741969_1145 (size=4469) 2024-11-26T19:14:21,745 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d23b79ada6d885c09c429afcb7484518/B of d23b79ada6d885c09c429afcb7484518 into 4296d4424a504e21a4a50ce8beba0baa(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T19:14:21,745 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d23b79ada6d885c09c429afcb7484518: 2024-11-26T19:14:21,745 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., storeName=d23b79ada6d885c09c429afcb7484518/B, priority=13, startTime=1732648461687; duration=0sec 2024-11-26T19:14:21,745 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T19:14:21,746 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d23b79ada6d885c09c429afcb7484518:B 2024-11-26T19:14:21,746 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T19:14:21,749 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T19:14:21,749 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1541): d23b79ada6d885c09c429afcb7484518/C is initiating minor compaction (all files) 2024-11-26T19:14:21,750 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d23b79ada6d885c09c429afcb7484518/C in TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. 2024-11-26T19:14:21,750 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/8f94eb4760c445b3825fc75efb49f2e0, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/9dad166e46f84480a73d811efbede772, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/d65e62fe075341f383973364f61ba044] into tmpdir=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp, totalSize=35.2 K 2024-11-26T19:14:21,751 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting 8f94eb4760c445b3825fc75efb49f2e0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732648459656 2024-11-26T19:14:21,752 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting 9dad166e46f84480a73d811efbede772, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732648459751 2024-11-26T19:14:21,753 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting d65e62fe075341f383973364f61ba044, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1732648460056 2024-11-26T19:14:21,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411263a1a9e908c5e4b91847310fac402566e_d23b79ada6d885c09c429afcb7484518 is 50, key is test_row_0/A:col10/1732648460726/Put/seqid=0 2024-11-26T19:14:21,781 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d23b79ada6d885c09c429afcb7484518#C#compaction#128 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T19:14:21,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741970_1146 (size=12154) 2024-11-26T19:14:21,782 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/C/f8158a46268440118a218d19eddc5eee is 50, key is test_row_0/C:col10/1732648460696/Put/seqid=0 2024-11-26T19:14:21,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:21,793 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411263a1a9e908c5e4b91847310fac402566e_d23b79ada6d885c09c429afcb7484518 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411263a1a9e908c5e4b91847310fac402566e_d23b79ada6d885c09c429afcb7484518 2024-11-26T19:14:21,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/A/2be2eaf6a9a749f38651fb10444f3051, store: [table=TestAcidGuarantees family=A region=d23b79ada6d885c09c429afcb7484518] 2024-11-26T19:14:21,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/A/2be2eaf6a9a749f38651fb10444f3051 is 175, key is test_row_0/A:col10/1732648460726/Put/seqid=0 2024-11-26T19:14:21,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741971_1147 (size=12104) 2024-11-26T19:14:21,827 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741972_1148 (size=30955) 2024-11-26T19:14:21,829 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=82, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/A/2be2eaf6a9a749f38651fb10444f3051 2024-11-26T19:14:21,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/B/bd5cacf6baaf429fb733a8a6f61e8711 is 50, key is test_row_0/B:col10/1732648460726/Put/seqid=0 2024-11-26T19:14:21,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741973_1149 (size=12001) 2024-11-26T19:14:21,858 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/B/bd5cacf6baaf429fb733a8a6f61e8711 2024-11-26T19:14:21,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/C/af6d53adee09486cb3a9c0fcf414330e is 50, key is test_row_0/C:col10/1732648460726/Put/seqid=0 2024-11-26T19:14:21,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741974_1150 (size=12001) 2024-11-26T19:14:21,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(8855): Flush requested on d23b79ada6d885c09c429afcb7484518 2024-11-26T19:14:21,874 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. as already flushing 2024-11-26T19:14:21,874 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/C/af6d53adee09486cb3a9c0fcf414330e 2024-11-26T19:14:21,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/A/2be2eaf6a9a749f38651fb10444f3051 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/2be2eaf6a9a749f38651fb10444f3051 2024-11-26T19:14:21,889 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/2be2eaf6a9a749f38651fb10444f3051, entries=150, sequenceid=82, filesize=30.2 K 2024-11-26T19:14:21,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/B/bd5cacf6baaf429fb733a8a6f61e8711 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/bd5cacf6baaf429fb733a8a6f61e8711 2024-11-26T19:14:21,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:21,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:21,897 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/bd5cacf6baaf429fb733a8a6f61e8711, entries=150, sequenceid=82, filesize=11.7 K 2024-11-26T19:14:21,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/C/af6d53adee09486cb3a9c0fcf414330e as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/af6d53adee09486cb3a9c0fcf414330e 2024-11-26T19:14:21,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:21,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:21,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:21,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:21,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:21,903 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/af6d53adee09486cb3a9c0fcf414330e, entries=150, sequenceid=82, filesize=11.7 K 2024-11-26T19:14:21,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:21,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:21,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:21,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:21,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:21,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:21,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:21,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:21,908 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=40.25 KB/41220 for d23b79ada6d885c09c429afcb7484518 in 188ms, sequenceid=82, compaction requested=false 2024-11-26T19:14:21,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2603): Flush status journal for d23b79ada6d885c09c429afcb7484518: 2024-11-26T19:14:21,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. 2024-11-26T19:14:21,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:21,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=39 2024-11-26T19:14:21,909 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d23b79ada6d885c09c429afcb7484518 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-26T19:14:21,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(8855): Flush requested on d23b79ada6d885c09c429afcb7484518 2024-11-26T19:14:21,909 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d23b79ada6d885c09c429afcb7484518, store=A 2024-11-26T19:14:21,909 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:14:21,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:21,909 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d23b79ada6d885c09c429afcb7484518, store=B 2024-11-26T19:14:21,909 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:14:21,909 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d23b79ada6d885c09c429afcb7484518, store=C 2024-11-26T19:14:21,909 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:14:21,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster(4169): Remote procedure done, pid=39 2024-11-26T19:14:21,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:21,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:21,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:21,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:21,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:21,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:21,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:21,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:21,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:21,915 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=38 2024-11-26T19:14:21,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:21,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:21,915 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1130 sec 2024-11-26T19:14:21,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:21,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:21,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:21,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:21,918 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees in 1.1230 sec 2024-11-26T19:14:21,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:21,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:21,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:21,921 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411260a5ccdabbbbb41db926574b7c019f129_d23b79ada6d885c09c429afcb7484518 is 50, key is test_row_0/A:col10/1732648461884/Put/seqid=0 2024-11-26T19:14:21,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:21,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:21,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:21,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:21,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:21,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:21,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:21,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:21,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:21,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:21,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:21,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:21,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:21,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:21,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741975_1151 (size=17034) 2024-11-26T19:14:21,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:21,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:21,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-26T19:14:21,933 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T19:14:21,933 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:21,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:21,933 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T19:14:21,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:21,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:21,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] procedure2.ProcedureExecutor(1139): Stored pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees 2024-11-26T19:14:21,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:21,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:21,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-26T19:14:21,936 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T19:14:21,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:21,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:21,937 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T19:14:21,937 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T19:14:21,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:21,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:21,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:21,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:21,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:21,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:21,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:21,943 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411260a5ccdabbbbb41db926574b7c019f129_d23b79ada6d885c09c429afcb7484518 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411260a5ccdabbbbb41db926574b7c019f129_d23b79ada6d885c09c429afcb7484518 2024-11-26T19:14:21,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:21,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:21,945 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/A/f0250d58167649bcb36eb05b988177ee, store: [table=TestAcidGuarantees family=A region=d23b79ada6d885c09c429afcb7484518] 2024-11-26T19:14:21,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:21,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:21,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:21,946 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/A/f0250d58167649bcb36eb05b988177ee is 175, key is test_row_0/A:col10/1732648461884/Put/seqid=0 2024-11-26T19:14:21,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:21,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:21,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:21,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:21,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:21,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:21,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:21,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:21,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:21,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:21,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:21,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:21,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:21,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:21,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:21,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:21,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:21,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:21,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:21,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:21,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:21,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:21,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:21,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:21,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:21,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:21,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:21,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:21,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:21,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:21,983 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=409, cellSize=18725, totalHeapSize=42776, min timestamp=1732648461963, max timestamp=1732648461979Num uniques 859; 2024-11-26T19:14:21,983 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=409, cellSize=18725, totalHeapSize=42776, min timestamp=1732648461963, max timestamp=1732648461979Num uniques 859; 2024-11-26T19:14:21,983 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=18725, suffix heap size=117064,new segment heap size=42776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=409 2024-11-26T19:14:21,983 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=18725, suffix heap size=117064,new segment heap size=42776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=409 2024-11-26T19:14:21,983 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=409, cellSize=18725, totalHeapSize=42776, min timestamp=1732648461963, max timestamp=1732648461979Num uniques 859; 2024-11-26T19:14:21,984 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=18725, suffix heap size=117064,new segment heap size=42776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=409 2024-11-26T19:14:21,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741976_1152 (size=48139) 2024-11-26T19:14:22,015 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:22,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39292 deadline: 1732648522002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:22,015 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:22,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39306 deadline: 1732648522002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:22,016 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:22,016 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:22,016 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:22,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39324 deadline: 1732648522004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:22,016 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:22,016 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:22,016 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:22,016 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:22,017 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:22,017 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:22,017 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:22,017 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:22,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39350 deadline: 1732648522015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:22,018 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:22,018 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:22,018 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:22,023 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:22,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39332 deadline: 1732648522020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:22,027 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:22,027 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:22,027 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:22,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-26T19:14:22,091 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-26T19:14:22,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. 2024-11-26T19:14:22,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. as already flushing 2024-11-26T19:14:22,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. 2024-11-26T19:14:22,092 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => d23b79ada6d885c09c429afcb7484518, NAME => 'TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:22,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => d23b79ada6d885c09c429afcb7484518, NAME => 'TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:22,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d23b79ada6d885c09c429afcb7484518, NAME => 'TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d23b79ada6d885c09c429afcb7484518, NAME => 'TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:22,126 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:22,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39292 deadline: 1732648522122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:22,127 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:22,127 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:22,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39306 deadline: 1732648522123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:22,127 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:22,127 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:22,127 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:22,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39324 deadline: 1732648522123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:22,127 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:22,127 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:22,127 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:22,127 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:22,128 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:22,128 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:22,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39350 deadline: 1732648522123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:22,128 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:22,128 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:22,128 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:22,128 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:22,139 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:22,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39332 deadline: 1732648522132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:22,139 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:22,139 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:22,139 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:22,142 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d23b79ada6d885c09c429afcb7484518#A#compaction#126 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T19:14:22,143 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/A/fba6c4a30c5945ae940f8322cad0734a is 175, key is test_row_0/A:col10/1732648460696/Put/seqid=0 2024-11-26T19:14:22,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741977_1153 (size=31058) 2024-11-26T19:14:22,209 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/C/f8158a46268440118a218d19eddc5eee as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/f8158a46268440118a218d19eddc5eee 2024-11-26T19:14:22,217 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d23b79ada6d885c09c429afcb7484518/C of d23b79ada6d885c09c429afcb7484518 into f8158a46268440118a218d19eddc5eee(size=11.8 K), total size for store is 23.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T19:14:22,217 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d23b79ada6d885c09c429afcb7484518: 2024-11-26T19:14:22,217 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., storeName=d23b79ada6d885c09c429afcb7484518/C, priority=13, startTime=1732648461687; duration=0sec 2024-11-26T19:14:22,217 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:14:22,218 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d23b79ada6d885c09c429afcb7484518:C 2024-11-26T19:14:22,245 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-26T19:14:22,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. 2024-11-26T19:14:22,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. as already flushing 2024-11-26T19:14:22,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. 2024-11-26T19:14:22,245 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => d23b79ada6d885c09c429afcb7484518, NAME => 'TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:22,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => d23b79ada6d885c09c429afcb7484518, NAME => 'TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:22,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d23b79ada6d885c09c429afcb7484518, NAME => 'TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d23b79ada6d885c09c429afcb7484518, NAME => 'TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:22,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-26T19:14:22,335 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:22,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39292 deadline: 1732648522332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:22,336 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:22,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39306 deadline: 1732648522332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:22,336 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:22,336 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:22,336 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:22,336 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:22,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39324 deadline: 1732648522332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:22,337 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:22,337 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:22,337 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:22,337 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:22,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39350 deadline: 1732648522332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:22,337 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:22,337 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:22,337 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:22,338 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:22,338 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:22,338 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:22,344 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:22,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39332 deadline: 1732648522342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:22,345 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:22,345 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:22,345 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:22,387 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=93, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/A/f0250d58167649bcb36eb05b988177ee 2024-11-26T19:14:22,400 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-26T19:14:22,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. 2024-11-26T19:14:22,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. as already flushing 2024-11-26T19:14:22,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. 2024-11-26T19:14:22,400 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => d23b79ada6d885c09c429afcb7484518, NAME => 'TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:22,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => d23b79ada6d885c09c429afcb7484518, NAME => 'TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:22,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d23b79ada6d885c09c429afcb7484518, NAME => 'TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d23b79ada6d885c09c429afcb7484518, NAME => 'TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:22,405 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/B/979c1f97799c400a851a925b214d4b7f is 50, key is test_row_0/B:col10/1732648461884/Put/seqid=0 2024-11-26T19:14:22,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741978_1154 (size=12001) 2024-11-26T19:14:22,414 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/B/979c1f97799c400a851a925b214d4b7f 2024-11-26T19:14:22,427 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/C/adcb77837cf541cf93981581ea852170 is 50, key is test_row_0/C:col10/1732648461884/Put/seqid=0 2024-11-26T19:14:22,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741979_1155 (size=12001) 2024-11-26T19:14:22,554 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-26T19:14:22,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. 2024-11-26T19:14:22,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. as already flushing 2024-11-26T19:14:22,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. 2024-11-26T19:14:22,554 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => d23b79ada6d885c09c429afcb7484518, NAME => 'TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:22,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => d23b79ada6d885c09c429afcb7484518, NAME => 'TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:22,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d23b79ada6d885c09c429afcb7484518, NAME => 'TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d23b79ada6d885c09c429afcb7484518, NAME => 'TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:22,555 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/A/fba6c4a30c5945ae940f8322cad0734a as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/fba6c4a30c5945ae940f8322cad0734a 2024-11-26T19:14:22,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-26T19:14:22,565 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d23b79ada6d885c09c429afcb7484518/A of d23b79ada6d885c09c429afcb7484518 into fba6c4a30c5945ae940f8322cad0734a(size=30.3 K), total size for store is 60.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T19:14:22,565 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d23b79ada6d885c09c429afcb7484518: 2024-11-26T19:14:22,565 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., storeName=d23b79ada6d885c09c429afcb7484518/A, priority=13, startTime=1732648461687; duration=0sec 2024-11-26T19:14:22,566 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:14:22,566 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d23b79ada6d885c09c429afcb7484518:A 2024-11-26T19:14:22,650 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:22,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39292 deadline: 1732648522642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:22,651 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:22,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39324 deadline: 1732648522642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:22,651 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:22,651 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:22,651 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:22,651 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:22,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39306 deadline: 1732648522642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:22,652 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:22,652 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:22,652 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:22,652 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:22,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39350 deadline: 1732648522642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:22,652 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:22,652 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:22,653 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:22,653 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:22,653 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:22,653 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:22,660 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:22,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39332 deadline: 1732648522652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:22,661 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:22,661 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:22,661 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:22,709 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-26T19:14:22,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. 2024-11-26T19:14:22,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. as already flushing 2024-11-26T19:14:22,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. 2024-11-26T19:14:22,709 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => d23b79ada6d885c09c429afcb7484518, NAME => 'TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:22,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => d23b79ada6d885c09c429afcb7484518, NAME => 'TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:22,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d23b79ada6d885c09c429afcb7484518, NAME => 'TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d23b79ada6d885c09c429afcb7484518, NAME => 'TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:22,832 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/C/adcb77837cf541cf93981581ea852170 2024-11-26T19:14:22,838 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/A/f0250d58167649bcb36eb05b988177ee as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/f0250d58167649bcb36eb05b988177ee 2024-11-26T19:14:22,845 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/f0250d58167649bcb36eb05b988177ee, entries=250, sequenceid=93, filesize=47.0 K 2024-11-26T19:14:22,846 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/B/979c1f97799c400a851a925b214d4b7f as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/979c1f97799c400a851a925b214d4b7f 2024-11-26T19:14:22,854 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/979c1f97799c400a851a925b214d4b7f, entries=150, sequenceid=93, filesize=11.7 K 2024-11-26T19:14:22,855 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/C/adcb77837cf541cf93981581ea852170 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/adcb77837cf541cf93981581ea852170 2024-11-26T19:14:22,863 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-26T19:14:22,863 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/adcb77837cf541cf93981581ea852170, entries=150, sequenceid=93, filesize=11.7 K 2024-11-26T19:14:22,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. 2024-11-26T19:14:22,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. as already flushing 2024-11-26T19:14:22,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. 2024-11-26T19:14:22,863 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => d23b79ada6d885c09c429afcb7484518, NAME => 'TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:22,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => d23b79ada6d885c09c429afcb7484518, NAME => 'TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:22,864 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=147.60 KB/151140 for d23b79ada6d885c09c429afcb7484518 in 956ms, sequenceid=93, compaction requested=true 2024-11-26T19:14:22,864 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d23b79ada6d885c09c429afcb7484518: 2024-11-26T19:14:22,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d23b79ada6d885c09c429afcb7484518, NAME => 'TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d23b79ada6d885c09c429afcb7484518, NAME => 'TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:22,864 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T19:14:22,865 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d23b79ada6d885c09c429afcb7484518:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T19:14:22,865 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:14:22,865 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T19:14:22,866 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110152 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T19:14:22,866 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1541): d23b79ada6d885c09c429afcb7484518/A is initiating minor compaction (all files) 2024-11-26T19:14:22,866 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d23b79ada6d885c09c429afcb7484518/A in TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. 2024-11-26T19:14:22,866 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d23b79ada6d885c09c429afcb7484518:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T19:14:22,866 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:14:22,866 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/fba6c4a30c5945ae940f8322cad0734a, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/2be2eaf6a9a749f38651fb10444f3051, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/f0250d58167649bcb36eb05b988177ee] into tmpdir=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp, totalSize=107.6 K 2024-11-26T19:14:22,866 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d23b79ada6d885c09c429afcb7484518:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T19:14:22,866 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T19:14:22,866 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. 2024-11-26T19:14:22,866 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. files: [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/fba6c4a30c5945ae940f8322cad0734a, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/2be2eaf6a9a749f38651fb10444f3051, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/f0250d58167649bcb36eb05b988177ee] 2024-11-26T19:14:22,866 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting fba6c4a30c5945ae940f8322cad0734a, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1732648460056 2024-11-26T19:14:22,867 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T19:14:22,867 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1541): d23b79ada6d885c09c429afcb7484518/B is initiating minor compaction (all files) 2024-11-26T19:14:22,867 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d23b79ada6d885c09c429afcb7484518/B in TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. 2024-11-26T19:14:22,867 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2be2eaf6a9a749f38651fb10444f3051, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1732648460726 2024-11-26T19:14:22,867 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/4296d4424a504e21a4a50ce8beba0baa, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/bd5cacf6baaf429fb733a8a6f61e8711, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/979c1f97799c400a851a925b214d4b7f] into tmpdir=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp, totalSize=35.3 K 2024-11-26T19:14:22,867 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting f0250d58167649bcb36eb05b988177ee, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1732648461882 2024-11-26T19:14:22,867 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting 4296d4424a504e21a4a50ce8beba0baa, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1732648460056 2024-11-26T19:14:22,868 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting bd5cacf6baaf429fb733a8a6f61e8711, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1732648460726 2024-11-26T19:14:22,868 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting 979c1f97799c400a851a925b214d4b7f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1732648461884 2024-11-26T19:14:22,885 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d23b79ada6d885c09c429afcb7484518] 2024-11-26T19:14:22,894 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d23b79ada6d885c09c429afcb7484518#B#compaction#135 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T19:14:22,894 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/B/ddcadf238f7b44328b5f67863ed8b5db is 50, key is test_row_0/B:col10/1732648461884/Put/seqid=0 2024-11-26T19:14:22,896 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241126afc3e793f7614863bc21e344d3344aa8_d23b79ada6d885c09c429afcb7484518 store=[table=TestAcidGuarantees family=A region=d23b79ada6d885c09c429afcb7484518] 2024-11-26T19:14:22,899 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241126afc3e793f7614863bc21e344d3344aa8_d23b79ada6d885c09c429afcb7484518, store=[table=TestAcidGuarantees family=A region=d23b79ada6d885c09c429afcb7484518] 2024-11-26T19:14:22,899 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126afc3e793f7614863bc21e344d3344aa8_d23b79ada6d885c09c429afcb7484518 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d23b79ada6d885c09c429afcb7484518] 2024-11-26T19:14:22,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741980_1156 (size=12207) 2024-11-26T19:14:22,916 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/B/ddcadf238f7b44328b5f67863ed8b5db as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/ddcadf238f7b44328b5f67863ed8b5db 2024-11-26T19:14:22,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741981_1157 (size=4469) 2024-11-26T19:14:22,923 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d23b79ada6d885c09c429afcb7484518#A#compaction#134 average throughput is 0.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T19:14:22,924 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d23b79ada6d885c09c429afcb7484518/B of d23b79ada6d885c09c429afcb7484518 into ddcadf238f7b44328b5f67863ed8b5db(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T19:14:22,924 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d23b79ada6d885c09c429afcb7484518: 2024-11-26T19:14:22,924 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/A/8674593c15d3428e8efbbc7cbaa9cea9 is 175, key is test_row_0/A:col10/1732648461884/Put/seqid=0 2024-11-26T19:14:22,924 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., storeName=d23b79ada6d885c09c429afcb7484518/B, priority=13, startTime=1732648462865; duration=0sec 2024-11-26T19:14:22,924 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T19:14:22,924 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d23b79ada6d885c09c429afcb7484518:B 2024-11-26T19:14:22,924 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T19:14:22,927 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T19:14:22,927 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1541): d23b79ada6d885c09c429afcb7484518/C is initiating minor compaction (all files) 2024-11-26T19:14:22,927 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d23b79ada6d885c09c429afcb7484518/C in TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. 2024-11-26T19:14:22,928 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/f8158a46268440118a218d19eddc5eee, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/af6d53adee09486cb3a9c0fcf414330e, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/adcb77837cf541cf93981581ea852170] into tmpdir=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp, totalSize=35.3 K 2024-11-26T19:14:22,928 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting f8158a46268440118a218d19eddc5eee, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1732648460056 2024-11-26T19:14:22,929 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting af6d53adee09486cb3a9c0fcf414330e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1732648460726 2024-11-26T19:14:22,929 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting adcb77837cf541cf93981581ea852170, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1732648461884 2024-11-26T19:14:22,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741982_1158 (size=31161) 2024-11-26T19:14:22,944 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/A/8674593c15d3428e8efbbc7cbaa9cea9 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/8674593c15d3428e8efbbc7cbaa9cea9 2024-11-26T19:14:22,951 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d23b79ada6d885c09c429afcb7484518#C#compaction#136 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T19:14:22,952 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/C/3e926e1f27b543698fc424bc4c4747b5 is 50, key is test_row_0/C:col10/1732648461884/Put/seqid=0 2024-11-26T19:14:22,957 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d23b79ada6d885c09c429afcb7484518/A of d23b79ada6d885c09c429afcb7484518 into 8674593c15d3428e8efbbc7cbaa9cea9(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T19:14:22,957 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d23b79ada6d885c09c429afcb7484518: 2024-11-26T19:14:22,957 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., storeName=d23b79ada6d885c09c429afcb7484518/A, priority=13, startTime=1732648462864; duration=0sec 2024-11-26T19:14:22,958 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:14:22,958 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d23b79ada6d885c09c429afcb7484518:A 2024-11-26T19:14:22,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741983_1159 (size=12207) 2024-11-26T19:14:22,970 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/C/3e926e1f27b543698fc424bc4c4747b5 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/3e926e1f27b543698fc424bc4c4747b5 2024-11-26T19:14:22,978 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d23b79ada6d885c09c429afcb7484518/C of d23b79ada6d885c09c429afcb7484518 into 3e926e1f27b543698fc424bc4c4747b5(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T19:14:22,978 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d23b79ada6d885c09c429afcb7484518: 2024-11-26T19:14:22,978 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., storeName=d23b79ada6d885c09c429afcb7484518/C, priority=13, startTime=1732648462866; duration=0sec 2024-11-26T19:14:22,978 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:14:22,978 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d23b79ada6d885c09c429afcb7484518:C 2024-11-26T19:14:23,016 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-26T19:14:23,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. 2024-11-26T19:14:23,017 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2902): Flushing d23b79ada6d885c09c429afcb7484518 3/3 column families, dataSize=147.60 KB heapSize=401.39 KB 2024-11-26T19:14:23,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d23b79ada6d885c09c429afcb7484518, store=A 2024-11-26T19:14:23,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T19:14:23,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d23b79ada6d885c09c429afcb7484518, store=B 2024-11-26T19:14:23,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T19:14:23,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d23b79ada6d885c09c429afcb7484518, store=C 2024-11-26T19:14:23,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T19:14:23,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411260e6b11aed79648af8be6dcae3a9e4d5b_d23b79ada6d885c09c429afcb7484518 is 50, key is test_row_0/A:col10/1732648462002/Put/seqid=0 2024-11-26T19:14:23,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-26T19:14:23,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741984_1160 (size=12154) 2024-11-26T19:14:23,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,093 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411260e6b11aed79648af8be6dcae3a9e4d5b_d23b79ada6d885c09c429afcb7484518 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411260e6b11aed79648af8be6dcae3a9e4d5b_d23b79ada6d885c09c429afcb7484518 2024-11-26T19:14:23,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/A/e95b58a058f248d6bd88a161f440e1fc, store: [table=TestAcidGuarantees family=A region=d23b79ada6d885c09c429afcb7484518] 2024-11-26T19:14:23,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/A/e95b58a058f248d6bd88a161f440e1fc is 175, key is test_row_0/A:col10/1732648462002/Put/seqid=0 2024-11-26T19:14:23,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741985_1161 (size=30955) 2024-11-26T19:14:23,128 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=132, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/A/e95b58a058f248d6bd88a161f440e1fc 2024-11-26T19:14:23,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/B/e8bfc2a4d27f4755b91f11dc8f98d904 is 50, key is test_row_0/B:col10/1732648462002/Put/seqid=0 2024-11-26T19:14:23,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741986_1162 (size=12001) 2024-11-26T19:14:23,160 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/B/e8bfc2a4d27f4755b91f11dc8f98d904 2024-11-26T19:14:23,167 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. as already flushing 2024-11-26T19:14:23,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(8855): Flush requested on d23b79ada6d885c09c429afcb7484518 2024-11-26T19:14:23,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/C/37a5a3dec23f4be88fb387744c79f1f2 is 50, key is test_row_0/C:col10/1732648462002/Put/seqid=0 2024-11-26T19:14:23,179 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:23,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39332 deadline: 1732648523172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:23,180 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:23,180 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:23,180 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:23,180 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:23,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39350 deadline: 1732648523173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:23,181 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:23,181 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:23,181 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:23,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741987_1163 (size=12001) 2024-11-26T19:14:23,186 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/C/37a5a3dec23f4be88fb387744c79f1f2 2024-11-26T19:14:23,186 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:23,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39306 deadline: 1732648523178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:23,187 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:23,187 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:23,187 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:23,187 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:23,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39292 deadline: 1732648523180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:23,187 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:23,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39324 deadline: 1732648523180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:23,188 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:23,188 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:23,188 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:23,188 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:23,188 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:23,188 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:23,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/A/e95b58a058f248d6bd88a161f440e1fc as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/e95b58a058f248d6bd88a161f440e1fc 2024-11-26T19:14:23,200 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/e95b58a058f248d6bd88a161f440e1fc, entries=150, sequenceid=132, filesize=30.2 K 2024-11-26T19:14:23,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/B/e8bfc2a4d27f4755b91f11dc8f98d904 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/e8bfc2a4d27f4755b91f11dc8f98d904 2024-11-26T19:14:23,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,208 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/e8bfc2a4d27f4755b91f11dc8f98d904, entries=150, sequenceid=132, filesize=11.7 K 2024-11-26T19:14:23,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/C/37a5a3dec23f4be88fb387744c79f1f2 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/37a5a3dec23f4be88fb387744c79f1f2 2024-11-26T19:14:23,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,214 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/37a5a3dec23f4be88fb387744c79f1f2, entries=150, sequenceid=132, filesize=11.7 K 2024-11-26T19:14:23,215 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~401.34 KB/410976, currentSize=46.96 KB/48090 for d23b79ada6d885c09c429afcb7484518 in 198ms, sequenceid=132, compaction requested=false 2024-11-26T19:14:23,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2603): Flush status journal for d23b79ada6d885c09c429afcb7484518: 2024-11-26T19:14:23,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. 2024-11-26T19:14:23,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=41 2024-11-26T19:14:23,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster(4169): Remote procedure done, pid=41 2024-11-26T19:14:23,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,218 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-11-26T19:14:23,218 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2790 sec 2024-11-26T19:14:23,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,222 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees in 1.2860 sec 2024-11-26T19:14:23,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(8855): Flush requested on d23b79ada6d885c09c429afcb7484518 2024-11-26T19:14:23,322 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d23b79ada6d885c09c429afcb7484518 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-26T19:14:23,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,323 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d23b79ada6d885c09c429afcb7484518, store=A 2024-11-26T19:14:23,323 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:14:23,323 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d23b79ada6d885c09c429afcb7484518, store=B 2024-11-26T19:14:23,324 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:14:23,324 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d23b79ada6d885c09c429afcb7484518, store=C 2024-11-26T19:14:23,324 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:14:23,343 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411269d17788213414966877eab02ed2f6192_d23b79ada6d885c09c429afcb7484518 is 50, key is test_row_0/A:col10/1732648463322/Put/seqid=0 2024-11-26T19:14:23,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741988_1164 (size=14794) 2024-11-26T19:14:23,356 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:23,362 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411269d17788213414966877eab02ed2f6192_d23b79ada6d885c09c429afcb7484518 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411269d17788213414966877eab02ed2f6192_d23b79ada6d885c09c429afcb7484518 2024-11-26T19:14:23,364 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/A/0f6d6c4fad404213a06e831a323e3c45, store: [table=TestAcidGuarantees family=A region=d23b79ada6d885c09c429afcb7484518] 2024-11-26T19:14:23,365 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/A/0f6d6c4fad404213a06e831a323e3c45 is 175, key is test_row_0/A:col10/1732648463322/Put/seqid=0 2024-11-26T19:14:23,368 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:23,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39324 deadline: 1732648523358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:23,368 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:23,368 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:23,368 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:23,373 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:23,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741989_1165 (size=39749) 2024-11-26T19:14:23,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39350 deadline: 1732648523365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:23,374 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:23,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39292 deadline: 1732648523367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:23,374 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:23,374 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:23,374 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:23,375 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:23,375 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:23,375 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:23,375 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=144, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/A/0f6d6c4fad404213a06e831a323e3c45 2024-11-26T19:14:23,375 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:23,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39306 deadline: 1732648523368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:23,376 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:23,376 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:23,376 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:23,385 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/B/e8788ae520b84ff9aff0be6675c1c5b8 is 50, key is test_row_0/B:col10/1732648463322/Put/seqid=0 2024-11-26T19:14:23,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741990_1166 (size=12151) 2024-11-26T19:14:23,478 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:23,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39324 deadline: 1732648523472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:23,479 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:23,479 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:23,479 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:23,482 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:23,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39350 deadline: 1732648523482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:23,483 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:23,483 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:23,483 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:23,483 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:23,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39306 deadline: 1732648523482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:23,483 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:23,483 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:23,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39292 deadline: 1732648523482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:23,483 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:23,483 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:23,484 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:23,484 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:23,484 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:23,687 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:23,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39324 deadline: 1732648523682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:23,688 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:23,688 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:23,688 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:23,693 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:23,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39350 deadline: 1732648523692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:23,693 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:23,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39306 deadline: 1732648523692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:23,693 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:23,694 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:23,694 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:23,694 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:23,694 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:23,694 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:23,694 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:23,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39292 deadline: 1732648523692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:23,694 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:23,694 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:23,694 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:23,792 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=144 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/B/e8788ae520b84ff9aff0be6675c1c5b8 2024-11-26T19:14:23,806 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/C/0386ac12602144ff9051f9a2b028b24e is 50, key is test_row_0/C:col10/1732648463322/Put/seqid=0 2024-11-26T19:14:23,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741991_1167 (size=12151) 2024-11-26T19:14:23,823 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=144 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/C/0386ac12602144ff9051f9a2b028b24e 2024-11-26T19:14:23,830 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/A/0f6d6c4fad404213a06e831a323e3c45 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/0f6d6c4fad404213a06e831a323e3c45 2024-11-26T19:14:23,837 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/0f6d6c4fad404213a06e831a323e3c45, entries=200, sequenceid=144, filesize=38.8 K 2024-11-26T19:14:23,838 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/B/e8788ae520b84ff9aff0be6675c1c5b8 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/e8788ae520b84ff9aff0be6675c1c5b8 2024-11-26T19:14:23,848 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/e8788ae520b84ff9aff0be6675c1c5b8, entries=150, sequenceid=144, filesize=11.9 K 2024-11-26T19:14:23,851 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/C/0386ac12602144ff9051f9a2b028b24e as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/0386ac12602144ff9051f9a2b028b24e 2024-11-26T19:14:23,856 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/0386ac12602144ff9051f9a2b028b24e, entries=150, sequenceid=144, filesize=11.9 K 2024-11-26T19:14:23,857 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for d23b79ada6d885c09c429afcb7484518 in 535ms, sequenceid=144, compaction requested=true 2024-11-26T19:14:23,858 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d23b79ada6d885c09c429afcb7484518: 2024-11-26T19:14:23,858 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d23b79ada6d885c09c429afcb7484518:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T19:14:23,858 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:14:23,858 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T19:14:23,858 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T19:14:23,858 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d23b79ada6d885c09c429afcb7484518:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T19:14:23,859 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101865 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T19:14:23,859 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1541): d23b79ada6d885c09c429afcb7484518/A is initiating minor compaction (all files) 2024-11-26T19:14:23,859 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d23b79ada6d885c09c429afcb7484518/A in TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. 2024-11-26T19:14:23,859 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/8674593c15d3428e8efbbc7cbaa9cea9, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/e95b58a058f248d6bd88a161f440e1fc, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/0f6d6c4fad404213a06e831a323e3c45] into tmpdir=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp, totalSize=99.5 K 2024-11-26T19:14:23,859 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. 2024-11-26T19:14:23,859 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. files: [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/8674593c15d3428e8efbbc7cbaa9cea9, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/e95b58a058f248d6bd88a161f440e1fc, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/0f6d6c4fad404213a06e831a323e3c45] 2024-11-26T19:14:23,860 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T19:14:23,860 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1541): d23b79ada6d885c09c429afcb7484518/B is initiating minor compaction (all files) 2024-11-26T19:14:23,860 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d23b79ada6d885c09c429afcb7484518/B in TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. 2024-11-26T19:14:23,860 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/ddcadf238f7b44328b5f67863ed8b5db, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/e8bfc2a4d27f4755b91f11dc8f98d904, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/e8788ae520b84ff9aff0be6675c1c5b8] into tmpdir=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp, totalSize=35.5 K 2024-11-26T19:14:23,860 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8674593c15d3428e8efbbc7cbaa9cea9, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1732648461884 2024-11-26T19:14:23,860 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:14:23,861 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d23b79ada6d885c09c429afcb7484518:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T19:14:23,861 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T19:14:23,861 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting ddcadf238f7b44328b5f67863ed8b5db, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1732648461884 2024-11-26T19:14:23,861 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting e95b58a058f248d6bd88a161f440e1fc, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732648462002 2024-11-26T19:14:23,861 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting e8bfc2a4d27f4755b91f11dc8f98d904, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732648462002 2024-11-26T19:14:23,862 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting e8788ae520b84ff9aff0be6675c1c5b8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1732648463176 2024-11-26T19:14:23,862 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0f6d6c4fad404213a06e831a323e3c45, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1732648463176 2024-11-26T19:14:23,883 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d23b79ada6d885c09c429afcb7484518] 2024-11-26T19:14:23,887 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d23b79ada6d885c09c429afcb7484518#B#compaction#144 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T19:14:23,888 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/B/ab08769c3ac146fd9c5e36d6e078de72 is 50, key is test_row_0/B:col10/1732648463322/Put/seqid=0 2024-11-26T19:14:23,897 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241126f99403f737444e5c8f176528d42e6dcc_d23b79ada6d885c09c429afcb7484518 store=[table=TestAcidGuarantees family=A region=d23b79ada6d885c09c429afcb7484518] 2024-11-26T19:14:23,899 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241126f99403f737444e5c8f176528d42e6dcc_d23b79ada6d885c09c429afcb7484518, store=[table=TestAcidGuarantees family=A region=d23b79ada6d885c09c429afcb7484518] 2024-11-26T19:14:23,900 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126f99403f737444e5c8f176528d42e6dcc_d23b79ada6d885c09c429afcb7484518 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d23b79ada6d885c09c429afcb7484518] 2024-11-26T19:14:23,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741993_1169 (size=4469) 2024-11-26T19:14:23,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741992_1168 (size=12459) 2024-11-26T19:14:23,937 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/B/ab08769c3ac146fd9c5e36d6e078de72 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/ab08769c3ac146fd9c5e36d6e078de72 2024-11-26T19:14:23,944 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d23b79ada6d885c09c429afcb7484518/B of d23b79ada6d885c09c429afcb7484518 into ab08769c3ac146fd9c5e36d6e078de72(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T19:14:23,944 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d23b79ada6d885c09c429afcb7484518: 2024-11-26T19:14:23,945 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., storeName=d23b79ada6d885c09c429afcb7484518/B, priority=13, startTime=1732648463858; duration=0sec 2024-11-26T19:14:23,945 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T19:14:23,945 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d23b79ada6d885c09c429afcb7484518:B 2024-11-26T19:14:23,945 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T19:14:23,947 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T19:14:23,947 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1541): d23b79ada6d885c09c429afcb7484518/C is initiating minor compaction (all files) 2024-11-26T19:14:23,948 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d23b79ada6d885c09c429afcb7484518/C in TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. 2024-11-26T19:14:23,948 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/3e926e1f27b543698fc424bc4c4747b5, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/37a5a3dec23f4be88fb387744c79f1f2, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/0386ac12602144ff9051f9a2b028b24e] into tmpdir=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp, totalSize=35.5 K 2024-11-26T19:14:23,948 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting 3e926e1f27b543698fc424bc4c4747b5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1732648461884 2024-11-26T19:14:23,949 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting 37a5a3dec23f4be88fb387744c79f1f2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732648462002 2024-11-26T19:14:23,949 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting 0386ac12602144ff9051f9a2b028b24e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1732648463176 2024-11-26T19:14:23,962 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d23b79ada6d885c09c429afcb7484518#C#compaction#145 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T19:14:23,963 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/C/5d07cf503cc1413bac0ecef9604f94bb is 50, key is test_row_0/C:col10/1732648463322/Put/seqid=0 2024-11-26T19:14:23,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741994_1170 (size=12459) 2024-11-26T19:14:23,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(8855): Flush requested on d23b79ada6d885c09c429afcb7484518 2024-11-26T19:14:23,996 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d23b79ada6d885c09c429afcb7484518 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-26T19:14:23,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d23b79ada6d885c09c429afcb7484518, store=A 2024-11-26T19:14:23,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T19:14:23,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d23b79ada6d885c09c429afcb7484518, store=B 2024-11-26T19:14:23,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T19:14:23,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d23b79ada6d885c09c429afcb7484518, store=C 2024-11-26T19:14:23,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T19:14:24,012 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126ec4feebfe1e148e1bce402ff2dd94e3d_d23b79ada6d885c09c429afcb7484518 is 50, key is test_row_0/A:col10/1732648463994/Put/seqid=0 2024-11-26T19:14:24,015 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:24,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39324 deadline: 1732648524011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:24,015 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:24,015 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:24,016 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:24,019 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:24,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39350 deadline: 1732648524013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:24,020 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:24,020 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:24,020 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:24,022 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:24,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39306 deadline: 1732648524015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:24,023 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:24,023 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:24,023 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:24,023 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:24,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39292 deadline: 1732648524015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:24,024 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:24,024 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:24,024 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:24,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741995_1171 (size=14794) 2024-11-26T19:14:24,043 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:24,048 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126ec4feebfe1e148e1bce402ff2dd94e3d_d23b79ada6d885c09c429afcb7484518 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126ec4feebfe1e148e1bce402ff2dd94e3d_d23b79ada6d885c09c429afcb7484518 2024-11-26T19:14:24,049 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/A/c33f837a479b44d99752ae594c8c2e20, store: [table=TestAcidGuarantees family=A region=d23b79ada6d885c09c429afcb7484518] 2024-11-26T19:14:24,050 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/A/c33f837a479b44d99752ae594c8c2e20 is 175, key is test_row_0/A:col10/1732648463994/Put/seqid=0 2024-11-26T19:14:24,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741996_1172 (size=39749) 2024-11-26T19:14:24,085 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=166, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/A/c33f837a479b44d99752ae594c8c2e20 2024-11-26T19:14:24,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-26T19:14:24,092 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T19:14:24,093 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T19:14:24,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-11-26T19:14:24,103 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/B/cabffbcc40e1472f998cec0d94529e4a is 50, key is test_row_0/B:col10/1732648463994/Put/seqid=0 2024-11-26T19:14:24,104 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T19:14:24,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-26T19:14:24,106 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T19:14:24,106 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T19:14:24,125 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:24,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39324 deadline: 1732648524122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:24,125 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:24,126 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:24,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39350 deadline: 1732648524122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:24,126 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:24,126 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:24,126 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:24,126 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:24,126 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:24,139 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:24,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39306 deadline: 1732648524132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:24,140 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:24,140 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:24,140 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:24,140 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:24,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39292 deadline: 1732648524132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:24,141 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:24,141 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:24,141 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:24,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741997_1173 (size=12151) 2024-11-26T19:14:24,194 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:24,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39332 deadline: 1732648524192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:24,195 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to fe76acf732b2,42785,1732648423970 for 'test_row_0' in d23b79ada6d885c09c429afcb7484518 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2177 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T19:14:24,196 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:24,196 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:24,196 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:24,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-26T19:14:24,260 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-26T19:14:24,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. 2024-11-26T19:14:24,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. as already flushing 2024-11-26T19:14:24,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. 2024-11-26T19:14:24,260 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => d23b79ada6d885c09c429afcb7484518, NAME => 'TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:24,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => d23b79ada6d885c09c429afcb7484518, NAME => 'TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:24,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d23b79ada6d885c09c429afcb7484518, NAME => 'TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d23b79ada6d885c09c429afcb7484518, NAME => 'TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:24,321 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d23b79ada6d885c09c429afcb7484518#A#compaction#143 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T19:14:24,322 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/A/01abae871b874ce8810a2a437c939142 is 175, key is test_row_0/A:col10/1732648463322/Put/seqid=0 2024-11-26T19:14:24,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741998_1174 (size=31413) 2024-11-26T19:14:24,333 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:24,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39324 deadline: 1732648524332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:24,334 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:24,334 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:24,334 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:24,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39350 deadline: 1732648524332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:24,334 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:24,334 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:24,334 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:24,334 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:24,343 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/A/01abae871b874ce8810a2a437c939142 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/01abae871b874ce8810a2a437c939142 2024-11-26T19:14:24,357 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:24,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39292 deadline: 1732648524352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:24,358 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:24,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39306 deadline: 1732648524352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:24,358 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:24,358 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:24,358 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:24,360 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:24,360 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:24,360 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:24,361 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d23b79ada6d885c09c429afcb7484518/A of d23b79ada6d885c09c429afcb7484518 into 01abae871b874ce8810a2a437c939142(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T19:14:24,361 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d23b79ada6d885c09c429afcb7484518: 2024-11-26T19:14:24,362 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., storeName=d23b79ada6d885c09c429afcb7484518/A, priority=13, startTime=1732648463858; duration=0sec 2024-11-26T19:14:24,362 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:14:24,362 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d23b79ada6d885c09c429afcb7484518:A 2024-11-26T19:14:24,402 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/C/5d07cf503cc1413bac0ecef9604f94bb as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/5d07cf503cc1413bac0ecef9604f94bb 2024-11-26T19:14:24,409 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d23b79ada6d885c09c429afcb7484518/C of d23b79ada6d885c09c429afcb7484518 into 5d07cf503cc1413bac0ecef9604f94bb(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T19:14:24,409 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d23b79ada6d885c09c429afcb7484518: 2024-11-26T19:14:24,409 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., storeName=d23b79ada6d885c09c429afcb7484518/C, priority=13, startTime=1732648463861; duration=0sec 2024-11-26T19:14:24,410 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:14:24,410 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d23b79ada6d885c09c429afcb7484518:C 2024-11-26T19:14:24,415 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-26T19:14:24,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. 2024-11-26T19:14:24,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. as already flushing 2024-11-26T19:14:24,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. 2024-11-26T19:14:24,416 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => d23b79ada6d885c09c429afcb7484518, NAME => 'TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:24,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => d23b79ada6d885c09c429afcb7484518, NAME => 'TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:24,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d23b79ada6d885c09c429afcb7484518, NAME => 'TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d23b79ada6d885c09c429afcb7484518, NAME => 'TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:24,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-26T19:14:24,566 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/B/cabffbcc40e1472f998cec0d94529e4a 2024-11-26T19:14:24,570 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-26T19:14:24,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. 2024-11-26T19:14:24,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. as already flushing 2024-11-26T19:14:24,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. 2024-11-26T19:14:24,570 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => d23b79ada6d885c09c429afcb7484518, NAME => 'TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:24,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => d23b79ada6d885c09c429afcb7484518, NAME => 'TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:24,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d23b79ada6d885c09c429afcb7484518, NAME => 'TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d23b79ada6d885c09c429afcb7484518, NAME => 'TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:24,580 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/C/231fd54ea85241b69dc73d8e4ed786ab is 50, key is test_row_0/C:col10/1732648463994/Put/seqid=0 2024-11-26T19:14:24,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741999_1175 (size=12151) 2024-11-26T19:14:24,614 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/C/231fd54ea85241b69dc73d8e4ed786ab 2024-11-26T19:14:24,629 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/A/c33f837a479b44d99752ae594c8c2e20 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/c33f837a479b44d99752ae594c8c2e20 2024-11-26T19:14:24,639 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/c33f837a479b44d99752ae594c8c2e20, entries=200, sequenceid=166, filesize=38.8 K 2024-11-26T19:14:24,642 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/B/cabffbcc40e1472f998cec0d94529e4a as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/cabffbcc40e1472f998cec0d94529e4a 2024-11-26T19:14:24,647 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:24,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39324 deadline: 1732648524642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:24,648 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:24,648 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:24,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39350 deadline: 1732648524642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:24,648 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:24,648 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/cabffbcc40e1472f998cec0d94529e4a, entries=150, sequenceid=166, filesize=11.9 K 2024-11-26T19:14:24,648 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:24,649 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:24,649 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:24,649 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:24,649 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/C/231fd54ea85241b69dc73d8e4ed786ab as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/231fd54ea85241b69dc73d8e4ed786ab 2024-11-26T19:14:24,660 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/231fd54ea85241b69dc73d8e4ed786ab, entries=150, sequenceid=166, filesize=11.9 K 2024-11-26T19:14:24,661 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for d23b79ada6d885c09c429afcb7484518 in 665ms, sequenceid=166, compaction requested=false 2024-11-26T19:14:24,661 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d23b79ada6d885c09c429afcb7484518: 2024-11-26T19:14:24,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(8855): Flush requested on d23b79ada6d885c09c429afcb7484518 2024-11-26T19:14:24,667 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d23b79ada6d885c09c429afcb7484518 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-26T19:14:24,667 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d23b79ada6d885c09c429afcb7484518, store=A 2024-11-26T19:14:24,667 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:14:24,667 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d23b79ada6d885c09c429afcb7484518, store=B 2024-11-26T19:14:24,667 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:14:24,667 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d23b79ada6d885c09c429afcb7484518, store=C 2024-11-26T19:14:24,667 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:14:24,720 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411267f9c22821b024c02bd8b82676d5c5585_d23b79ada6d885c09c429afcb7484518 is 50, key is test_row_0/A:col10/1732648464663/Put/seqid=0 2024-11-26T19:14:24,724 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-26T19:14:24,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. 2024-11-26T19:14:24,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. as already flushing 2024-11-26T19:14:24,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. 2024-11-26T19:14:24,725 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => d23b79ada6d885c09c429afcb7484518, NAME => 'TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:24,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => d23b79ada6d885c09c429afcb7484518, NAME => 'TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:24,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d23b79ada6d885c09c429afcb7484518, NAME => 'TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d23b79ada6d885c09c429afcb7484518, NAME => 'TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:24,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-26T19:14:24,751 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:24,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39292 deadline: 1732648524744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:24,752 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:24,752 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:24,752 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:24,758 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:24,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39306 deadline: 1732648524751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:24,759 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:24,759 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:24,759 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:24,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742000_1176 (size=12304) 2024-11-26T19:14:24,867 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:24,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39292 deadline: 1732648524862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:24,868 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:24,868 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:24,868 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:24,868 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:24,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39306 deadline: 1732648524862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:24,869 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:24,869 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:24,869 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:24,879 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-26T19:14:24,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. 2024-11-26T19:14:24,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. as already flushing 2024-11-26T19:14:24,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. 2024-11-26T19:14:24,880 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => d23b79ada6d885c09c429afcb7484518, NAME => 'TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:24,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => d23b79ada6d885c09c429afcb7484518, NAME => 'TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:24,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d23b79ada6d885c09c429afcb7484518, NAME => 'TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d23b79ada6d885c09c429afcb7484518, NAME => 'TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:25,033 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-26T19:14:25,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. 2024-11-26T19:14:25,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. as already flushing 2024-11-26T19:14:25,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. 2024-11-26T19:14:25,034 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => d23b79ada6d885c09c429afcb7484518, NAME => 'TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:25,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => d23b79ada6d885c09c429afcb7484518, NAME => 'TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:25,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d23b79ada6d885c09c429afcb7484518, NAME => 'TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d23b79ada6d885c09c429afcb7484518, NAME => 'TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:25,076 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:25,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39292 deadline: 1732648525072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:25,077 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:25,077 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:25,077 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:25,083 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:25,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39306 deadline: 1732648525082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:25,086 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:25,086 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:25,086 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:25,154 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:25,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39324 deadline: 1732648525152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:25,155 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:25,155 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:25,155 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:25,167 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:25,170 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:25,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39350 deadline: 1732648525164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:25,171 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:25,171 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:25,171 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:25,173 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411267f9c22821b024c02bd8b82676d5c5585_d23b79ada6d885c09c429afcb7484518 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411267f9c22821b024c02bd8b82676d5c5585_d23b79ada6d885c09c429afcb7484518 2024-11-26T19:14:25,174 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/A/4c8b5e337b094684834986dd87123dbe, store: [table=TestAcidGuarantees family=A region=d23b79ada6d885c09c429afcb7484518] 2024-11-26T19:14:25,175 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/A/4c8b5e337b094684834986dd87123dbe is 175, key is test_row_0/A:col10/1732648464663/Put/seqid=0 2024-11-26T19:14:25,188 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-26T19:14:25,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. 2024-11-26T19:14:25,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. as already flushing 2024-11-26T19:14:25,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. 2024-11-26T19:14:25,188 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => d23b79ada6d885c09c429afcb7484518, NAME => 'TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:25,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => d23b79ada6d885c09c429afcb7484518, NAME => 'TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:25,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d23b79ada6d885c09c429afcb7484518, NAME => 'TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d23b79ada6d885c09c429afcb7484518, NAME => 'TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:25,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742001_1177 (size=31105) 2024-11-26T19:14:25,227 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=180, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/A/4c8b5e337b094684834986dd87123dbe 2024-11-26T19:14:25,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-26T19:14:25,251 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/B/7d9ce28c36d2421f8134b71ca2e95216 is 50, key is test_row_0/B:col10/1732648464663/Put/seqid=0 2024-11-26T19:14:25,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742002_1178 (size=12151) 2024-11-26T19:14:25,342 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-26T19:14:25,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. 2024-11-26T19:14:25,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. as already flushing 2024-11-26T19:14:25,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. 2024-11-26T19:14:25,343 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => d23b79ada6d885c09c429afcb7484518, NAME => 'TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:25,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => d23b79ada6d885c09c429afcb7484518, NAME => 'TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:25,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d23b79ada6d885c09c429afcb7484518, NAME => 'TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d23b79ada6d885c09c429afcb7484518, NAME => 'TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:25,382 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:25,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39292 deadline: 1732648525381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:25,383 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:25,383 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:25,383 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:25,395 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:25,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39306 deadline: 1732648525392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:25,395 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:25,395 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:25,396 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:25,497 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-26T19:14:25,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. 2024-11-26T19:14:25,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. as already flushing 2024-11-26T19:14:25,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. 2024-11-26T19:14:25,497 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => d23b79ada6d885c09c429afcb7484518, NAME => 'TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:25,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => d23b79ada6d885c09c429afcb7484518, NAME => 'TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:25,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d23b79ada6d885c09c429afcb7484518, NAME => 'TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d23b79ada6d885c09c429afcb7484518, NAME => 'TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:25,651 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-26T19:14:25,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. 2024-11-26T19:14:25,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. as already flushing 2024-11-26T19:14:25,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. 2024-11-26T19:14:25,652 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => d23b79ada6d885c09c429afcb7484518, NAME => 'TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:25,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => d23b79ada6d885c09c429afcb7484518, NAME => 'TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:25,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d23b79ada6d885c09c429afcb7484518, NAME => 'TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d23b79ada6d885c09c429afcb7484518, NAME => 'TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:25,698 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=180 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/B/7d9ce28c36d2421f8134b71ca2e95216 2024-11-26T19:14:25,709 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/C/2957ce24dd854bf0906556c76e7fa025 is 50, key is test_row_0/C:col10/1732648464663/Put/seqid=0 2024-11-26T19:14:25,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742003_1179 (size=12151) 2024-11-26T19:14:25,744 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=180 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/C/2957ce24dd854bf0906556c76e7fa025 2024-11-26T19:14:25,752 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/A/4c8b5e337b094684834986dd87123dbe as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/4c8b5e337b094684834986dd87123dbe 2024-11-26T19:14:25,761 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/4c8b5e337b094684834986dd87123dbe, entries=150, sequenceid=180, filesize=30.4 K 2024-11-26T19:14:25,762 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/B/7d9ce28c36d2421f8134b71ca2e95216 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/7d9ce28c36d2421f8134b71ca2e95216 2024-11-26T19:14:25,775 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/7d9ce28c36d2421f8134b71ca2e95216, entries=150, sequenceid=180, filesize=11.9 K 2024-11-26T19:14:25,776 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/C/2957ce24dd854bf0906556c76e7fa025 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/2957ce24dd854bf0906556c76e7fa025 2024-11-26T19:14:25,789 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/2957ce24dd854bf0906556c76e7fa025, entries=150, sequenceid=180, filesize=11.9 K 2024-11-26T19:14:25,790 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for d23b79ada6d885c09c429afcb7484518 in 1123ms, sequenceid=180, compaction requested=true 2024-11-26T19:14:25,790 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d23b79ada6d885c09c429afcb7484518: 2024-11-26T19:14:25,790 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d23b79ada6d885c09c429afcb7484518:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T19:14:25,790 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:14:25,790 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T19:14:25,790 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d23b79ada6d885c09c429afcb7484518:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T19:14:25,790 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T19:14:25,790 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d23b79ada6d885c09c429afcb7484518:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T19:14:25,790 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-26T19:14:25,790 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T19:14:25,791 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102267 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T19:14:25,791 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1541): d23b79ada6d885c09c429afcb7484518/A is initiating minor compaction (all files) 2024-11-26T19:14:25,791 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T19:14:25,791 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d23b79ada6d885c09c429afcb7484518/A in TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. 2024-11-26T19:14:25,791 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1541): d23b79ada6d885c09c429afcb7484518/B is initiating minor compaction (all files) 2024-11-26T19:14:25,792 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d23b79ada6d885c09c429afcb7484518/B in TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. 2024-11-26T19:14:25,792 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/01abae871b874ce8810a2a437c939142, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/c33f837a479b44d99752ae594c8c2e20, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/4c8b5e337b094684834986dd87123dbe] into tmpdir=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp, totalSize=99.9 K 2024-11-26T19:14:25,792 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. 2024-11-26T19:14:25,792 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/ab08769c3ac146fd9c5e36d6e078de72, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/cabffbcc40e1472f998cec0d94529e4a, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/7d9ce28c36d2421f8134b71ca2e95216] into tmpdir=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp, totalSize=35.9 K 2024-11-26T19:14:25,792 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. files: [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/01abae871b874ce8810a2a437c939142, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/c33f837a479b44d99752ae594c8c2e20, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/4c8b5e337b094684834986dd87123dbe] 2024-11-26T19:14:25,793 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting 01abae871b874ce8810a2a437c939142, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1732648463176 2024-11-26T19:14:25,794 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting ab08769c3ac146fd9c5e36d6e078de72, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1732648463176 2024-11-26T19:14:25,794 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting c33f837a479b44d99752ae594c8c2e20, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1732648463354 2024-11-26T19:14:25,794 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4c8b5e337b094684834986dd87123dbe, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1732648464011 2024-11-26T19:14:25,795 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting cabffbcc40e1472f998cec0d94529e4a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1732648463354 2024-11-26T19:14:25,795 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting 7d9ce28c36d2421f8134b71ca2e95216, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1732648464011 2024-11-26T19:14:25,804 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-26T19:14:25,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. 2024-11-26T19:14:25,805 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing d23b79ada6d885c09c429afcb7484518 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-26T19:14:25,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d23b79ada6d885c09c429afcb7484518, store=A 2024-11-26T19:14:25,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:14:25,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d23b79ada6d885c09c429afcb7484518, store=B 2024-11-26T19:14:25,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:14:25,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d23b79ada6d885c09c429afcb7484518, store=C 2024-11-26T19:14:25,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:14:25,806 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d23b79ada6d885c09c429afcb7484518] 2024-11-26T19:14:25,811 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d23b79ada6d885c09c429afcb7484518#B#compaction#153 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T19:14:25,811 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/B/de440b23eb3e4a6e906face8234ac12c is 50, key is test_row_0/B:col10/1732648464663/Put/seqid=0 2024-11-26T19:14:25,817 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241126f8bbbe88325f4e93b5478964f7e54c86_d23b79ada6d885c09c429afcb7484518 store=[table=TestAcidGuarantees family=A region=d23b79ada6d885c09c429afcb7484518] 2024-11-26T19:14:25,819 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241126f8bbbe88325f4e93b5478964f7e54c86_d23b79ada6d885c09c429afcb7484518, store=[table=TestAcidGuarantees family=A region=d23b79ada6d885c09c429afcb7484518] 2024-11-26T19:14:25,819 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126f8bbbe88325f4e93b5478964f7e54c86_d23b79ada6d885c09c429afcb7484518 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d23b79ada6d885c09c429afcb7484518] 2024-11-26T19:14:25,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126292ac7f09cbc4d7fa8638298acc199d2_d23b79ada6d885c09c429afcb7484518 is 50, key is test_row_0/A:col10/1732648464741/Put/seqid=0 2024-11-26T19:14:25,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742004_1180 (size=12561) 2024-11-26T19:14:25,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(8855): Flush requested on d23b79ada6d885c09c429afcb7484518 2024-11-26T19:14:25,921 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. as already flushing 2024-11-26T19:14:25,927 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/B/de440b23eb3e4a6e906face8234ac12c as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/de440b23eb3e4a6e906face8234ac12c 2024-11-26T19:14:25,936 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d23b79ada6d885c09c429afcb7484518/B of d23b79ada6d885c09c429afcb7484518 into de440b23eb3e4a6e906face8234ac12c(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T19:14:25,936 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d23b79ada6d885c09c429afcb7484518: 2024-11-26T19:14:25,936 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., storeName=d23b79ada6d885c09c429afcb7484518/B, priority=13, startTime=1732648465790; duration=0sec 2024-11-26T19:14:25,936 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T19:14:25,936 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d23b79ada6d885c09c429afcb7484518:B 2024-11-26T19:14:25,936 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T19:14:25,939 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T19:14:25,939 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1541): d23b79ada6d885c09c429afcb7484518/C is initiating minor compaction (all files) 2024-11-26T19:14:25,939 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d23b79ada6d885c09c429afcb7484518/C in TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. 2024-11-26T19:14:25,939 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/5d07cf503cc1413bac0ecef9604f94bb, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/231fd54ea85241b69dc73d8e4ed786ab, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/2957ce24dd854bf0906556c76e7fa025] into tmpdir=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp, totalSize=35.9 K 2024-11-26T19:14:25,941 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting 5d07cf503cc1413bac0ecef9604f94bb, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1732648463176 2024-11-26T19:14:25,942 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting 231fd54ea85241b69dc73d8e4ed786ab, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1732648463354 2024-11-26T19:14:25,943 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting 2957ce24dd854bf0906556c76e7fa025, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1732648464011 2024-11-26T19:14:25,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742006_1182 (size=12304) 2024-11-26T19:14:25,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742005_1181 (size=4469) 2024-11-26T19:14:25,979 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:25,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39292 deadline: 1732648525968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:25,979 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:25,980 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:25,980 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:25,982 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d23b79ada6d885c09c429afcb7484518#C#compaction#155 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T19:14:25,983 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/C/d199d3678a9a4e7bbb84f40c070d22ef is 50, key is test_row_0/C:col10/1732648464663/Put/seqid=0 2024-11-26T19:14:25,988 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:25,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39306 deadline: 1732648525978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:25,989 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:25,989 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:25,989 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:26,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742007_1183 (size=12561) 2024-11-26T19:14:26,033 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/C/d199d3678a9a4e7bbb84f40c070d22ef as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/d199d3678a9a4e7bbb84f40c070d22ef 2024-11-26T19:14:26,046 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d23b79ada6d885c09c429afcb7484518/C of d23b79ada6d885c09c429afcb7484518 into d199d3678a9a4e7bbb84f40c070d22ef(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T19:14:26,046 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d23b79ada6d885c09c429afcb7484518: 2024-11-26T19:14:26,046 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., storeName=d23b79ada6d885c09c429afcb7484518/C, priority=13, startTime=1732648465790; duration=0sec 2024-11-26T19:14:26,046 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:14:26,046 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d23b79ada6d885c09c429afcb7484518:C 2024-11-26T19:14:26,086 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:26,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39292 deadline: 1732648526082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:26,089 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:26,089 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:26,089 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:26,096 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:26,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39306 deadline: 1732648526091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:26,097 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:26,097 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:26,097 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:26,164 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:26,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39324 deadline: 1732648526161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:26,165 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to fe76acf732b2,42785,1732648423970 for 'test_row_2' in d23b79ada6d885c09c429afcb7484518 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2153 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T19:14:26,165 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:26,165 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:26,165 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:26,182 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:26,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39350 deadline: 1732648526182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:26,183 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to fe76acf732b2,42785,1732648423970 for 'test_row_0' in d23b79ada6d885c09c429afcb7484518 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T19:14:26,183 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:26,184 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:26,184 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:26,214 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:26,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39332 deadline: 1732648526212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:26,214 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to fe76acf732b2,42785,1732648423970 for 'test_row_0' in d23b79ada6d885c09c429afcb7484518 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4196 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T19:14:26,215 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:26,215 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:26,215 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:26,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-26T19:14:26,295 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:26,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39292 deadline: 1732648526292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:26,296 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:26,296 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:26,296 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:26,302 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:26,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39306 deadline: 1732648526301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:26,303 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:26,303 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:26,303 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:26,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:26,368 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126292ac7f09cbc4d7fa8638298acc199d2_d23b79ada6d885c09c429afcb7484518 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126292ac7f09cbc4d7fa8638298acc199d2_d23b79ada6d885c09c429afcb7484518 2024-11-26T19:14:26,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/A/3dde213ba3e84412810eaef5d9bb1299, store: [table=TestAcidGuarantees family=A region=d23b79ada6d885c09c429afcb7484518] 2024-11-26T19:14:26,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/A/3dde213ba3e84412810eaef5d9bb1299 is 175, key is test_row_0/A:col10/1732648464741/Put/seqid=0 2024-11-26T19:14:26,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742008_1184 (size=31105) 2024-11-26T19:14:26,375 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d23b79ada6d885c09c429afcb7484518#A#compaction#152 average throughput is 0.04 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T19:14:26,376 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/A/2868ae3397044c76bea4e911b36b2a6d is 175, key is test_row_0/A:col10/1732648464663/Put/seqid=0 2024-11-26T19:14:26,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742009_1185 (size=31515) 2024-11-26T19:14:26,388 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/A/2868ae3397044c76bea4e911b36b2a6d as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/2868ae3397044c76bea4e911b36b2a6d 2024-11-26T19:14:26,398 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d23b79ada6d885c09c429afcb7484518/A of d23b79ada6d885c09c429afcb7484518 into 2868ae3397044c76bea4e911b36b2a6d(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T19:14:26,398 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d23b79ada6d885c09c429afcb7484518: 2024-11-26T19:14:26,398 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., storeName=d23b79ada6d885c09c429afcb7484518/A, priority=13, startTime=1732648465790; duration=0sec 2024-11-26T19:14:26,399 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:14:26,399 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d23b79ada6d885c09c429afcb7484518:A 2024-11-26T19:14:26,606 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:26,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39292 deadline: 1732648526601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:26,607 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:26,607 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:26,607 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:26,612 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:26,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39306 deadline: 1732648526612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:26,613 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:26,613 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:26,613 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:26,775 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=200, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/A/3dde213ba3e84412810eaef5d9bb1299 2024-11-26T19:14:26,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/B/009b75cfad224716865d1635886facf3 is 50, key is test_row_0/B:col10/1732648464741/Put/seqid=0 2024-11-26T19:14:26,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742010_1186 (size=12151) 2024-11-26T19:14:26,815 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=200 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/B/009b75cfad224716865d1635886facf3 2024-11-26T19:14:26,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/C/e5d2a4534a834c0b964fdb830090962a is 50, key is test_row_0/C:col10/1732648464741/Put/seqid=0 2024-11-26T19:14:26,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742011_1187 (size=12151) 2024-11-26T19:14:27,115 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:27,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39292 deadline: 1732648527112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:27,115 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:27,115 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:27,116 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:27,126 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:27,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39306 deadline: 1732648527123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:27,127 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:27,127 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:27,127 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:27,254 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=200 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/C/e5d2a4534a834c0b964fdb830090962a 2024-11-26T19:14:27,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/A/3dde213ba3e84412810eaef5d9bb1299 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/3dde213ba3e84412810eaef5d9bb1299 2024-11-26T19:14:27,264 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/3dde213ba3e84412810eaef5d9bb1299, entries=150, sequenceid=200, filesize=30.4 K 2024-11-26T19:14:27,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/B/009b75cfad224716865d1635886facf3 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/009b75cfad224716865d1635886facf3 2024-11-26T19:14:27,276 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/009b75cfad224716865d1635886facf3, entries=150, sequenceid=200, filesize=11.9 K 2024-11-26T19:14:27,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/C/e5d2a4534a834c0b964fdb830090962a as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/e5d2a4534a834c0b964fdb830090962a 2024-11-26T19:14:27,283 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/e5d2a4534a834c0b964fdb830090962a, entries=150, sequenceid=200, filesize=11.9 K 2024-11-26T19:14:27,286 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for d23b79ada6d885c09c429afcb7484518 in 1481ms, sequenceid=200, compaction requested=false 2024-11-26T19:14:27,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for d23b79ada6d885c09c429afcb7484518: 2024-11-26T19:14:27,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. 2024-11-26T19:14:27,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-11-26T19:14:27,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-11-26T19:14:27,290 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-26T19:14:27,290 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.1820 sec 2024-11-26T19:14:27,292 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 3.1980 sec 2024-11-26T19:14:27,844 INFO [master/fe76acf732b2:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-26T19:14:27,844 INFO [master/fe76acf732b2:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-26T19:14:28,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(8855): Flush requested on d23b79ada6d885c09c429afcb7484518 2024-11-26T19:14:28,126 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d23b79ada6d885c09c429afcb7484518 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-26T19:14:28,126 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d23b79ada6d885c09c429afcb7484518, store=A 2024-11-26T19:14:28,126 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:14:28,126 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d23b79ada6d885c09c429afcb7484518, store=B 2024-11-26T19:14:28,126 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:14:28,126 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d23b79ada6d885c09c429afcb7484518, store=C 2024-11-26T19:14:28,126 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:14:28,151 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126e189e8991d33425fad185dfba3b59ef3_d23b79ada6d885c09c429afcb7484518 is 50, key is test_row_0/A:col10/1732648465966/Put/seqid=0 2024-11-26T19:14:28,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742012_1188 (size=14794) 2024-11-26T19:14:28,201 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:28,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39324 deadline: 1732648528192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:28,202 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:28,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39292 deadline: 1732648528193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:28,202 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to fe76acf732b2,42785,1732648423970 for 'test_row_2' in d23b79ada6d885c09c429afcb7484518 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4190 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T19:14:28,202 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:28,202 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:28,202 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:28,202 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:28,202 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:28,202 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:28,202 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:28,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39306 deadline: 1732648528196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:28,203 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:28,203 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:28,203 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:28,203 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:28,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39350 deadline: 1732648528202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:28,203 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to fe76acf732b2,42785,1732648423970 for 'test_row_0' in d23b79ada6d885c09c429afcb7484518 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4190 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T19:14:28,204 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:28,204 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:28,204 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:28,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-26T19:14:28,262 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T19:14:28,263 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T19:14:28,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-11-26T19:14:28,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-26T19:14:28,267 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T19:14:28,268 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T19:14:28,268 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T19:14:28,312 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:28,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39292 deadline: 1732648528311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:28,313 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:28,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39306 deadline: 1732648528311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:28,313 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:28,313 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:28,313 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:28,313 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:28,313 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:28,313 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:28,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-26T19:14:28,421 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-26T19:14:28,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. 2024-11-26T19:14:28,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. as already flushing 2024-11-26T19:14:28,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. 2024-11-26T19:14:28,422 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => d23b79ada6d885c09c429afcb7484518, NAME => 'TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:28,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => d23b79ada6d885c09c429afcb7484518, NAME => 'TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:28,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d23b79ada6d885c09c429afcb7484518, NAME => 'TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d23b79ada6d885c09c429afcb7484518, NAME => 'TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:28,529 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:28,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39292 deadline: 1732648528522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:28,529 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:28,530 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:28,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39306 deadline: 1732648528522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:28,530 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:28,530 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:28,530 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:28,530 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:28,530 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:28,578 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-26T19:14:28,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. 2024-11-26T19:14:28,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. as already flushing 2024-11-26T19:14:28,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. 2024-11-26T19:14:28,578 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => d23b79ada6d885c09c429afcb7484518, NAME => 'TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:28,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => d23b79ada6d885c09c429afcb7484518, NAME => 'TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:28,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d23b79ada6d885c09c429afcb7484518, NAME => 'TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d23b79ada6d885c09c429afcb7484518, NAME => 'TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:28,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-26T19:14:28,593 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:28,598 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126e189e8991d33425fad185dfba3b59ef3_d23b79ada6d885c09c429afcb7484518 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126e189e8991d33425fad185dfba3b59ef3_d23b79ada6d885c09c429afcb7484518 2024-11-26T19:14:28,600 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/A/e04f1b3aa4e343f387ae7eec9cf362b5, store: [table=TestAcidGuarantees family=A region=d23b79ada6d885c09c429afcb7484518] 2024-11-26T19:14:28,600 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/A/e04f1b3aa4e343f387ae7eec9cf362b5 is 175, key is test_row_0/A:col10/1732648465966/Put/seqid=0 2024-11-26T19:14:28,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742013_1189 (size=39749) 2024-11-26T19:14:28,648 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=216, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/A/e04f1b3aa4e343f387ae7eec9cf362b5 2024-11-26T19:14:28,667 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/B/40ebf4ff13404ea0b4f841745cd5421f is 50, key is test_row_0/B:col10/1732648465966/Put/seqid=0 2024-11-26T19:14:28,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742014_1190 (size=12151) 2024-11-26T19:14:28,732 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-26T19:14:28,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. 2024-11-26T19:14:28,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. as already flushing 2024-11-26T19:14:28,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. 2024-11-26T19:14:28,732 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => d23b79ada6d885c09c429afcb7484518, NAME => 'TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:28,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => d23b79ada6d885c09c429afcb7484518, NAME => 'TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:28,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d23b79ada6d885c09c429afcb7484518, NAME => 'TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d23b79ada6d885c09c429afcb7484518, NAME => 'TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:28,845 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:28,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39306 deadline: 1732648528844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:28,846 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:28,846 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:28,846 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:28,846 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:28,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39292 deadline: 1732648528844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:28,847 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:28,847 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:28,847 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:28,885 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-26T19:14:28,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. 2024-11-26T19:14:28,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. as already flushing 2024-11-26T19:14:28,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. 2024-11-26T19:14:28,886 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => d23b79ada6d885c09c429afcb7484518, NAME => 'TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:28,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => d23b79ada6d885c09c429afcb7484518, NAME => 'TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:28,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d23b79ada6d885c09c429afcb7484518, NAME => 'TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d23b79ada6d885c09c429afcb7484518, NAME => 'TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:28,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-26T19:14:29,039 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-26T19:14:29,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. 2024-11-26T19:14:29,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. as already flushing 2024-11-26T19:14:29,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. 2024-11-26T19:14:29,040 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => d23b79ada6d885c09c429afcb7484518, NAME => 'TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:29,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => d23b79ada6d885c09c429afcb7484518, NAME => 'TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:29,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d23b79ada6d885c09c429afcb7484518, NAME => 'TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d23b79ada6d885c09c429afcb7484518, NAME => 'TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:29,076 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/B/40ebf4ff13404ea0b4f841745cd5421f 2024-11-26T19:14:29,101 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/C/6ce2b385a67945b1bb95fbd0ba51abf1 is 50, key is test_row_0/C:col10/1732648465966/Put/seqid=0 2024-11-26T19:14:29,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742015_1191 (size=12151) 2024-11-26T19:14:29,119 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/C/6ce2b385a67945b1bb95fbd0ba51abf1 2024-11-26T19:14:29,127 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/A/e04f1b3aa4e343f387ae7eec9cf362b5 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/e04f1b3aa4e343f387ae7eec9cf362b5 2024-11-26T19:14:29,144 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/e04f1b3aa4e343f387ae7eec9cf362b5, entries=200, sequenceid=216, filesize=38.8 K 2024-11-26T19:14:29,153 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/B/40ebf4ff13404ea0b4f841745cd5421f as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/40ebf4ff13404ea0b4f841745cd5421f 2024-11-26T19:14:29,163 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/40ebf4ff13404ea0b4f841745cd5421f, entries=150, sequenceid=216, filesize=11.9 K 2024-11-26T19:14:29,165 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/C/6ce2b385a67945b1bb95fbd0ba51abf1 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/6ce2b385a67945b1bb95fbd0ba51abf1 2024-11-26T19:14:29,174 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/6ce2b385a67945b1bb95fbd0ba51abf1, entries=150, sequenceid=216, filesize=11.9 K 2024-11-26T19:14:29,176 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for d23b79ada6d885c09c429afcb7484518 in 1051ms, sequenceid=216, compaction requested=true 2024-11-26T19:14:29,176 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d23b79ada6d885c09c429afcb7484518: 2024-11-26T19:14:29,176 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T19:14:29,176 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d23b79ada6d885c09c429afcb7484518:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T19:14:29,176 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:14:29,176 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T19:14:29,177 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d23b79ada6d885c09c429afcb7484518:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T19:14:29,177 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:14:29,177 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d23b79ada6d885c09c429afcb7484518:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T19:14:29,177 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T19:14:29,185 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102369 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T19:14:29,185 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1541): d23b79ada6d885c09c429afcb7484518/A is initiating minor compaction (all files) 2024-11-26T19:14:29,185 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d23b79ada6d885c09c429afcb7484518/A in TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. 2024-11-26T19:14:29,185 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/2868ae3397044c76bea4e911b36b2a6d, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/3dde213ba3e84412810eaef5d9bb1299, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/e04f1b3aa4e343f387ae7eec9cf362b5] into tmpdir=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp, totalSize=100.0 K 2024-11-26T19:14:29,185 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. 2024-11-26T19:14:29,185 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. files: [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/2868ae3397044c76bea4e911b36b2a6d, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/3dde213ba3e84412810eaef5d9bb1299, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/e04f1b3aa4e343f387ae7eec9cf362b5] 2024-11-26T19:14:29,185 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T19:14:29,186 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1541): d23b79ada6d885c09c429afcb7484518/B is initiating minor compaction (all files) 2024-11-26T19:14:29,186 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d23b79ada6d885c09c429afcb7484518/B in TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. 2024-11-26T19:14:29,186 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/de440b23eb3e4a6e906face8234ac12c, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/009b75cfad224716865d1635886facf3, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/40ebf4ff13404ea0b4f841745cd5421f] into tmpdir=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp, totalSize=36.0 K 2024-11-26T19:14:29,186 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2868ae3397044c76bea4e911b36b2a6d, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1732648464011 2024-11-26T19:14:29,186 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting de440b23eb3e4a6e906face8234ac12c, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1732648464011 2024-11-26T19:14:29,186 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3dde213ba3e84412810eaef5d9bb1299, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1732648464729 2024-11-26T19:14:29,187 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting 009b75cfad224716865d1635886facf3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1732648464729 2024-11-26T19:14:29,187 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting e04f1b3aa4e343f387ae7eec9cf362b5, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1732648465962 2024-11-26T19:14:29,188 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting 40ebf4ff13404ea0b4f841745cd5421f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1732648465966 2024-11-26T19:14:29,193 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-26T19:14:29,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. 2024-11-26T19:14:29,195 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing d23b79ada6d885c09c429afcb7484518 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-26T19:14:29,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d23b79ada6d885c09c429afcb7484518, store=A 2024-11-26T19:14:29,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:14:29,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d23b79ada6d885c09c429afcb7484518, store=B 2024-11-26T19:14:29,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:14:29,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d23b79ada6d885c09c429afcb7484518, store=C 2024-11-26T19:14:29,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:14:29,213 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d23b79ada6d885c09c429afcb7484518] 2024-11-26T19:14:29,217 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d23b79ada6d885c09c429afcb7484518#B#compaction#162 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T19:14:29,218 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/B/69dec79ce76a43199606362ff99df531 is 50, key is test_row_0/B:col10/1732648465966/Put/seqid=0 2024-11-26T19:14:29,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411265b11e856cd734b669de56eb444a54589_d23b79ada6d885c09c429afcb7484518 is 50, key is test_row_0/A:col10/1732648468191/Put/seqid=0 2024-11-26T19:14:29,235 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411261c03dc5c302f412b9f75296bb85f56e2_d23b79ada6d885c09c429afcb7484518 store=[table=TestAcidGuarantees family=A region=d23b79ada6d885c09c429afcb7484518] 2024-11-26T19:14:29,238 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411261c03dc5c302f412b9f75296bb85f56e2_d23b79ada6d885c09c429afcb7484518, store=[table=TestAcidGuarantees family=A region=d23b79ada6d885c09c429afcb7484518] 2024-11-26T19:14:29,238 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411261c03dc5c302f412b9f75296bb85f56e2_d23b79ada6d885c09c429afcb7484518 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d23b79ada6d885c09c429afcb7484518] 2024-11-26T19:14:29,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742017_1193 (size=12304) 2024-11-26T19:14:29,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742016_1192 (size=12663) 2024-11-26T19:14:29,278 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/B/69dec79ce76a43199606362ff99df531 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/69dec79ce76a43199606362ff99df531 2024-11-26T19:14:29,289 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d23b79ada6d885c09c429afcb7484518/B of d23b79ada6d885c09c429afcb7484518 into 69dec79ce76a43199606362ff99df531(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T19:14:29,289 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d23b79ada6d885c09c429afcb7484518: 2024-11-26T19:14:29,289 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., storeName=d23b79ada6d885c09c429afcb7484518/B, priority=13, startTime=1732648469176; duration=0sec 2024-11-26T19:14:29,289 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T19:14:29,289 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d23b79ada6d885c09c429afcb7484518:B 2024-11-26T19:14:29,289 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T19:14:29,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742018_1194 (size=4469) 2024-11-26T19:14:29,295 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T19:14:29,295 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1541): d23b79ada6d885c09c429afcb7484518/C is initiating minor compaction (all files) 2024-11-26T19:14:29,295 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d23b79ada6d885c09c429afcb7484518/C in TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. 2024-11-26T19:14:29,295 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/d199d3678a9a4e7bbb84f40c070d22ef, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/e5d2a4534a834c0b964fdb830090962a, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/6ce2b385a67945b1bb95fbd0ba51abf1] into tmpdir=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp, totalSize=36.0 K 2024-11-26T19:14:29,296 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting d199d3678a9a4e7bbb84f40c070d22ef, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1732648464011 2024-11-26T19:14:29,296 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting e5d2a4534a834c0b964fdb830090962a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1732648464729 2024-11-26T19:14:29,297 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting 6ce2b385a67945b1bb95fbd0ba51abf1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1732648465966 2024-11-26T19:14:29,309 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d23b79ada6d885c09c429afcb7484518#A#compaction#161 average throughput is 0.25 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T19:14:29,310 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/A/96a41b21e7a942d793f3f66eabe8d937 is 175, key is test_row_0/A:col10/1732648465966/Put/seqid=0 2024-11-26T19:14:29,335 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d23b79ada6d885c09c429afcb7484518#C#compaction#164 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T19:14:29,335 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/C/d1145a17dfc64bed940314e9924e86e6 is 50, key is test_row_0/C:col10/1732648465966/Put/seqid=0 2024-11-26T19:14:29,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742019_1195 (size=31617) 2024-11-26T19:14:29,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(8855): Flush requested on d23b79ada6d885c09c429afcb7484518 2024-11-26T19:14:29,355 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. as already flushing 2024-11-26T19:14:29,357 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/A/96a41b21e7a942d793f3f66eabe8d937 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/96a41b21e7a942d793f3f66eabe8d937 2024-11-26T19:14:29,366 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d23b79ada6d885c09c429afcb7484518/A of d23b79ada6d885c09c429afcb7484518 into 96a41b21e7a942d793f3f66eabe8d937(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T19:14:29,366 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d23b79ada6d885c09c429afcb7484518: 2024-11-26T19:14:29,366 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., storeName=d23b79ada6d885c09c429afcb7484518/A, priority=13, startTime=1732648469176; duration=0sec 2024-11-26T19:14:29,366 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:14:29,366 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d23b79ada6d885c09c429afcb7484518:A 2024-11-26T19:14:29,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742020_1196 (size=12663) 2024-11-26T19:14:29,381 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/C/d1145a17dfc64bed940314e9924e86e6 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/d1145a17dfc64bed940314e9924e86e6 2024-11-26T19:14:29,393 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d23b79ada6d885c09c429afcb7484518/C of d23b79ada6d885c09c429afcb7484518 into d1145a17dfc64bed940314e9924e86e6(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T19:14:29,393 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d23b79ada6d885c09c429afcb7484518: 2024-11-26T19:14:29,393 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., storeName=d23b79ada6d885c09c429afcb7484518/C, priority=13, startTime=1732648469177; duration=0sec 2024-11-26T19:14:29,394 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:14:29,394 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d23b79ada6d885c09c429afcb7484518:C 2024-11-26T19:14:29,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-26T19:14:29,411 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:29,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39306 deadline: 1732648529402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:29,412 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:29,412 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:29,412 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:29,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39292 deadline: 1732648529403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:29,412 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:29,413 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:29,413 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:29,413 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:29,526 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:29,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39292 deadline: 1732648529523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:29,527 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:29,527 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:29,527 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:29,527 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:29,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39306 deadline: 1732648529523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:29,528 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:29,528 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:29,528 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:29,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:29,675 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411265b11e856cd734b669de56eb444a54589_d23b79ada6d885c09c429afcb7484518 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411265b11e856cd734b669de56eb444a54589_d23b79ada6d885c09c429afcb7484518 2024-11-26T19:14:29,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/A/a0b8c1e8d50e41039eb047ec9f582621, store: [table=TestAcidGuarantees family=A region=d23b79ada6d885c09c429afcb7484518] 2024-11-26T19:14:29,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/A/a0b8c1e8d50e41039eb047ec9f582621 is 175, key is test_row_0/A:col10/1732648468191/Put/seqid=0 2024-11-26T19:14:29,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742021_1197 (size=31105) 2024-11-26T19:14:29,687 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=235, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/A/a0b8c1e8d50e41039eb047ec9f582621 2024-11-26T19:14:29,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/B/fd349611854a47aab9acdf3ef75aff32 is 50, key is test_row_0/B:col10/1732648468191/Put/seqid=0 2024-11-26T19:14:29,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742022_1198 (size=12151) 2024-11-26T19:14:29,704 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/B/fd349611854a47aab9acdf3ef75aff32 2024-11-26T19:14:29,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/C/98dcb5015a0444e59511871e6b221032 is 50, key is test_row_0/C:col10/1732648468191/Put/seqid=0 2024-11-26T19:14:29,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742023_1199 (size=12151) 2024-11-26T19:14:29,733 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:29,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39292 deadline: 1732648529732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:29,734 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:29,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39306 deadline: 1732648529732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:29,734 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:29,734 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:29,734 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:29,734 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:29,734 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:29,734 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:30,043 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:30,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39292 deadline: 1732648530042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:30,044 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:30,044 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:30,044 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:30,044 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:30,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39306 deadline: 1732648530042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:30,044 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:30,044 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:30,044 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:30,120 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/C/98dcb5015a0444e59511871e6b221032 2024-11-26T19:14:30,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/A/a0b8c1e8d50e41039eb047ec9f582621 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/a0b8c1e8d50e41039eb047ec9f582621 2024-11-26T19:14:30,130 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/a0b8c1e8d50e41039eb047ec9f582621, entries=150, sequenceid=235, filesize=30.4 K 2024-11-26T19:14:30,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/B/fd349611854a47aab9acdf3ef75aff32 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/fd349611854a47aab9acdf3ef75aff32 2024-11-26T19:14:30,137 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/fd349611854a47aab9acdf3ef75aff32, entries=150, sequenceid=235, filesize=11.9 K 2024-11-26T19:14:30,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/C/98dcb5015a0444e59511871e6b221032 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/98dcb5015a0444e59511871e6b221032 2024-11-26T19:14:30,144 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/98dcb5015a0444e59511871e6b221032, entries=150, sequenceid=235, filesize=11.9 K 2024-11-26T19:14:30,145 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for d23b79ada6d885c09c429afcb7484518 in 949ms, sequenceid=235, compaction requested=false 2024-11-26T19:14:30,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for d23b79ada6d885c09c429afcb7484518: 2024-11-26T19:14:30,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. 2024-11-26T19:14:30,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-11-26T19:14:30,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-11-26T19:14:30,148 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-26T19:14:30,148 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8780 sec 2024-11-26T19:14:30,151 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 1.8860 sec 2024-11-26T19:14:30,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(8855): Flush requested on d23b79ada6d885c09c429afcb7484518 2024-11-26T19:14:30,264 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d23b79ada6d885c09c429afcb7484518 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-26T19:14:30,264 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d23b79ada6d885c09c429afcb7484518, store=A 2024-11-26T19:14:30,264 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:14:30,264 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d23b79ada6d885c09c429afcb7484518, store=B 2024-11-26T19:14:30,264 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:14:30,264 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d23b79ada6d885c09c429afcb7484518, store=C 2024-11-26T19:14:30,264 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:14:30,273 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112688f59e068c5e4e1a95478a99d2582198_d23b79ada6d885c09c429afcb7484518 is 50, key is test_row_0/A:col10/1732648470262/Put/seqid=0 2024-11-26T19:14:30,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742024_1200 (size=14794) 2024-11-26T19:14:30,277 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:30,281 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112688f59e068c5e4e1a95478a99d2582198_d23b79ada6d885c09c429afcb7484518 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112688f59e068c5e4e1a95478a99d2582198_d23b79ada6d885c09c429afcb7484518 2024-11-26T19:14:30,282 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/A/bbc93e0962c14986ad2d6f3a781eea95, store: [table=TestAcidGuarantees family=A region=d23b79ada6d885c09c429afcb7484518] 2024-11-26T19:14:30,283 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/A/bbc93e0962c14986ad2d6f3a781eea95 is 175, key is test_row_0/A:col10/1732648470262/Put/seqid=0 2024-11-26T19:14:30,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742025_1201 (size=39749) 2024-11-26T19:14:30,345 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:30,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39332 deadline: 1732648530337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:30,346 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:30,346 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:30,346 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:30,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-26T19:14:30,412 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T19:14:30,413 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T19:14:30,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-11-26T19:14:30,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-26T19:14:30,415 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T19:14:30,415 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T19:14:30,415 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T19:14:30,455 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:30,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39332 deadline: 1732648530452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:30,455 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:30,455 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:30,455 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:30,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-26T19:14:30,553 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:30,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39292 deadline: 1732648530551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:30,553 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:30,554 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:30,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39306 deadline: 1732648530551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:30,554 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:30,554 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:30,554 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:30,554 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:30,554 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:30,568 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-26T19:14:30,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. 2024-11-26T19:14:30,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. as already flushing 2024-11-26T19:14:30,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. 2024-11-26T19:14:30,568 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => d23b79ada6d885c09c429afcb7484518, NAME => 'TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:30,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => d23b79ada6d885c09c429afcb7484518, NAME => 'TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:30,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d23b79ada6d885c09c429afcb7484518, NAME => 'TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d23b79ada6d885c09c429afcb7484518, NAME => 'TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:30,668 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:30,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39332 deadline: 1732648530661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:30,669 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:30,669 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:30,669 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:30,691 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=252, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/A/bbc93e0962c14986ad2d6f3a781eea95 2024-11-26T19:14:30,702 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/B/8dc74111e6c3484abb5ce11d12bbd4ae is 50, key is test_row_0/B:col10/1732648470262/Put/seqid=0 2024-11-26T19:14:30,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742026_1202 (size=12151) 2024-11-26T19:14:30,712 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/B/8dc74111e6c3484abb5ce11d12bbd4ae 2024-11-26T19:14:30,723 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-26T19:14:30,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. 2024-11-26T19:14:30,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. as already flushing 2024-11-26T19:14:30,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. 2024-11-26T19:14:30,723 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => d23b79ada6d885c09c429afcb7484518, NAME => 'TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:30,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => d23b79ada6d885c09c429afcb7484518, NAME => 'TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:30,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d23b79ada6d885c09c429afcb7484518, NAME => 'TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d23b79ada6d885c09c429afcb7484518, NAME => 'TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:30,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-26T19:14:30,739 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/C/5a80909cae87476ba8b94d49331429cb is 50, key is test_row_0/C:col10/1732648470262/Put/seqid=0 2024-11-26T19:14:30,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742027_1203 (size=12151) 2024-11-26T19:14:30,878 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-26T19:14:30,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. 2024-11-26T19:14:30,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. as already flushing 2024-11-26T19:14:30,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. 2024-11-26T19:14:30,879 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => d23b79ada6d885c09c429afcb7484518, NAME => 'TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:30,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => d23b79ada6d885c09c429afcb7484518, NAME => 'TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:30,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d23b79ada6d885c09c429afcb7484518, NAME => 'TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d23b79ada6d885c09c429afcb7484518, NAME => 'TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:30,973 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:30,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39332 deadline: 1732648530972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:30,973 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:30,973 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:30,974 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:31,032 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-26T19:14:31,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. 2024-11-26T19:14:31,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. as already flushing 2024-11-26T19:14:31,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. 2024-11-26T19:14:31,032 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => d23b79ada6d885c09c429afcb7484518, NAME => 'TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:31,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => d23b79ada6d885c09c429afcb7484518, NAME => 'TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:31,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d23b79ada6d885c09c429afcb7484518, NAME => 'TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d23b79ada6d885c09c429afcb7484518, NAME => 'TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:31,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-26T19:14:31,158 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/C/5a80909cae87476ba8b94d49331429cb 2024-11-26T19:14:31,170 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/A/bbc93e0962c14986ad2d6f3a781eea95 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/bbc93e0962c14986ad2d6f3a781eea95 2024-11-26T19:14:31,184 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/bbc93e0962c14986ad2d6f3a781eea95, entries=200, sequenceid=252, filesize=38.8 K 2024-11-26T19:14:31,185 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-26T19:14:31,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. 2024-11-26T19:14:31,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. as already flushing 2024-11-26T19:14:31,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. 2024-11-26T19:14:31,185 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/B/8dc74111e6c3484abb5ce11d12bbd4ae as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/8dc74111e6c3484abb5ce11d12bbd4ae 2024-11-26T19:14:31,186 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => d23b79ada6d885c09c429afcb7484518, NAME => 'TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:31,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => d23b79ada6d885c09c429afcb7484518, NAME => 'TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:31,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d23b79ada6d885c09c429afcb7484518, NAME => 'TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d23b79ada6d885c09c429afcb7484518, NAME => 'TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:31,196 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/8dc74111e6c3484abb5ce11d12bbd4ae, entries=150, sequenceid=252, filesize=11.9 K 2024-11-26T19:14:31,198 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/C/5a80909cae87476ba8b94d49331429cb as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/5a80909cae87476ba8b94d49331429cb 2024-11-26T19:14:31,203 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/5a80909cae87476ba8b94d49331429cb, entries=150, sequenceid=252, filesize=11.9 K 2024-11-26T19:14:31,205 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for d23b79ada6d885c09c429afcb7484518 in 942ms, sequenceid=252, compaction requested=true 2024-11-26T19:14:31,205 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d23b79ada6d885c09c429afcb7484518: 2024-11-26T19:14:31,205 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d23b79ada6d885c09c429afcb7484518:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T19:14:31,205 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:14:31,205 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d23b79ada6d885c09c429afcb7484518:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T19:14:31,205 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:14:31,205 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T19:14:31,205 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d23b79ada6d885c09c429afcb7484518:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T19:14:31,205 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T19:14:31,205 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T19:14:31,207 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102471 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T19:14:31,207 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1541): d23b79ada6d885c09c429afcb7484518/A is initiating minor compaction (all files) 2024-11-26T19:14:31,207 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d23b79ada6d885c09c429afcb7484518/A in TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. 2024-11-26T19:14:31,207 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/96a41b21e7a942d793f3f66eabe8d937, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/a0b8c1e8d50e41039eb047ec9f582621, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/bbc93e0962c14986ad2d6f3a781eea95] into tmpdir=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp, totalSize=100.1 K 2024-11-26T19:14:31,207 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. 2024-11-26T19:14:31,207 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. files: [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/96a41b21e7a942d793f3f66eabe8d937, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/a0b8c1e8d50e41039eb047ec9f582621, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/bbc93e0962c14986ad2d6f3a781eea95] 2024-11-26T19:14:31,207 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T19:14:31,207 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1541): d23b79ada6d885c09c429afcb7484518/B is initiating minor compaction (all files) 2024-11-26T19:14:31,207 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d23b79ada6d885c09c429afcb7484518/B in TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. 2024-11-26T19:14:31,207 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/69dec79ce76a43199606362ff99df531, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/fd349611854a47aab9acdf3ef75aff32, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/8dc74111e6c3484abb5ce11d12bbd4ae] into tmpdir=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp, totalSize=36.1 K 2024-11-26T19:14:31,208 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting 96a41b21e7a942d793f3f66eabe8d937, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1732648465966 2024-11-26T19:14:31,208 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting 69dec79ce76a43199606362ff99df531, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1732648465966 2024-11-26T19:14:31,208 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting fd349611854a47aab9acdf3ef75aff32, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1732648468146 2024-11-26T19:14:31,208 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting a0b8c1e8d50e41039eb047ec9f582621, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1732648468146 2024-11-26T19:14:31,209 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting 8dc74111e6c3484abb5ce11d12bbd4ae, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1732648469390 2024-11-26T19:14:31,209 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting bbc93e0962c14986ad2d6f3a781eea95, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1732648469387 2024-11-26T19:14:31,221 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d23b79ada6d885c09c429afcb7484518#B#compaction#170 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T19:14:31,222 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/B/96e1560c0b164ff5bcc212fd25486acf is 50, key is test_row_0/B:col10/1732648470262/Put/seqid=0 2024-11-26T19:14:31,237 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d23b79ada6d885c09c429afcb7484518] 2024-11-26T19:14:31,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742028_1204 (size=12765) 2024-11-26T19:14:31,261 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411268a7a24e150af40a08b25ad66238f5f06_d23b79ada6d885c09c429afcb7484518 store=[table=TestAcidGuarantees family=A region=d23b79ada6d885c09c429afcb7484518] 2024-11-26T19:14:31,262 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/B/96e1560c0b164ff5bcc212fd25486acf as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/96e1560c0b164ff5bcc212fd25486acf 2024-11-26T19:14:31,264 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411268a7a24e150af40a08b25ad66238f5f06_d23b79ada6d885c09c429afcb7484518, store=[table=TestAcidGuarantees family=A region=d23b79ada6d885c09c429afcb7484518] 2024-11-26T19:14:31,264 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411268a7a24e150af40a08b25ad66238f5f06_d23b79ada6d885c09c429afcb7484518 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d23b79ada6d885c09c429afcb7484518] 2024-11-26T19:14:31,271 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d23b79ada6d885c09c429afcb7484518/B of d23b79ada6d885c09c429afcb7484518 into 96e1560c0b164ff5bcc212fd25486acf(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T19:14:31,271 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d23b79ada6d885c09c429afcb7484518: 2024-11-26T19:14:31,271 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., storeName=d23b79ada6d885c09c429afcb7484518/B, priority=13, startTime=1732648471205; duration=0sec 2024-11-26T19:14:31,271 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T19:14:31,271 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d23b79ada6d885c09c429afcb7484518:B 2024-11-26T19:14:31,271 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T19:14:31,273 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T19:14:31,273 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1541): d23b79ada6d885c09c429afcb7484518/C is initiating minor compaction (all files) 2024-11-26T19:14:31,273 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d23b79ada6d885c09c429afcb7484518/C in TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. 2024-11-26T19:14:31,273 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/d1145a17dfc64bed940314e9924e86e6, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/98dcb5015a0444e59511871e6b221032, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/5a80909cae87476ba8b94d49331429cb] into tmpdir=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp, totalSize=36.1 K 2024-11-26T19:14:31,274 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting d1145a17dfc64bed940314e9924e86e6, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1732648465966 2024-11-26T19:14:31,275 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting 98dcb5015a0444e59511871e6b221032, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1732648468146 2024-11-26T19:14:31,275 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting 5a80909cae87476ba8b94d49331429cb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1732648469390 2024-11-26T19:14:31,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742029_1205 (size=4469) 2024-11-26T19:14:31,300 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d23b79ada6d885c09c429afcb7484518#A#compaction#171 average throughput is 0.39 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T19:14:31,301 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/A/8f1d34e65e1947c49f150925f1a3cd26 is 175, key is test_row_0/A:col10/1732648470262/Put/seqid=0 2024-11-26T19:14:31,304 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d23b79ada6d885c09c429afcb7484518#C#compaction#172 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T19:14:31,305 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/C/a97470276177497c951a479e55ec275a is 50, key is test_row_0/C:col10/1732648470262/Put/seqid=0 2024-11-26T19:14:31,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742031_1207 (size=12765) 2024-11-26T19:14:31,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742030_1206 (size=31719) 2024-11-26T19:14:31,339 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-26T19:14:31,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. 2024-11-26T19:14:31,340 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing d23b79ada6d885c09c429afcb7484518 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-26T19:14:31,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d23b79ada6d885c09c429afcb7484518, store=A 2024-11-26T19:14:31,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:14:31,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d23b79ada6d885c09c429afcb7484518, store=B 2024-11-26T19:14:31,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:14:31,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d23b79ada6d885c09c429afcb7484518, store=C 2024-11-26T19:14:31,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:14:31,345 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/A/8f1d34e65e1947c49f150925f1a3cd26 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/8f1d34e65e1947c49f150925f1a3cd26 2024-11-26T19:14:31,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126a94e70e4714d4488b125023f21878042_d23b79ada6d885c09c429afcb7484518 is 50, key is test_row_0/A:col10/1732648470335/Put/seqid=0 2024-11-26T19:14:31,353 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d23b79ada6d885c09c429afcb7484518/A of d23b79ada6d885c09c429afcb7484518 into 8f1d34e65e1947c49f150925f1a3cd26(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T19:14:31,353 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d23b79ada6d885c09c429afcb7484518: 2024-11-26T19:14:31,353 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., storeName=d23b79ada6d885c09c429afcb7484518/A, priority=13, startTime=1732648471205; duration=0sec 2024-11-26T19:14:31,353 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:14:31,353 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d23b79ada6d885c09c429afcb7484518:A 2024-11-26T19:14:31,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742032_1208 (size=12454) 2024-11-26T19:14:31,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(8855): Flush requested on d23b79ada6d885c09c429afcb7484518 2024-11-26T19:14:31,485 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. as already flushing 2024-11-26T19:14:31,547 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:31,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39332 deadline: 1732648531547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:31,548 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:31,548 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:31,548 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:31,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-26T19:14:31,562 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:31,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39306 deadline: 1732648531561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:31,563 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to fe76acf732b2,42785,1732648423970 for 'test_row_0' in d23b79ada6d885c09c429afcb7484518 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2161 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T19:14:31,563 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:31,563 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:31,563 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:31,573 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:31,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39292 deadline: 1732648531571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:31,573 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to fe76acf732b2,42785,1732648423970 for 'test_row_2' in d23b79ada6d885c09c429afcb7484518 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2170 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T19:14:31,574 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:31,574 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:31,574 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:31,652 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:31,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39332 deadline: 1732648531651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:31,652 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:31,652 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:31,652 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:31,730 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/C/a97470276177497c951a479e55ec275a as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/a97470276177497c951a479e55ec275a 2024-11-26T19:14:31,737 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d23b79ada6d885c09c429afcb7484518/C of d23b79ada6d885c09c429afcb7484518 into a97470276177497c951a479e55ec275a(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T19:14:31,738 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d23b79ada6d885c09c429afcb7484518: 2024-11-26T19:14:31,738 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., storeName=d23b79ada6d885c09c429afcb7484518/C, priority=13, startTime=1732648471205; duration=0sec 2024-11-26T19:14:31,738 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:14:31,738 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d23b79ada6d885c09c429afcb7484518:C 2024-11-26T19:14:31,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:31,761 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126a94e70e4714d4488b125023f21878042_d23b79ada6d885c09c429afcb7484518 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126a94e70e4714d4488b125023f21878042_d23b79ada6d885c09c429afcb7484518 2024-11-26T19:14:31,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/A/fbafe2f643924acca20660e5b3ceac02, store: [table=TestAcidGuarantees family=A region=d23b79ada6d885c09c429afcb7484518] 2024-11-26T19:14:31,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/A/fbafe2f643924acca20660e5b3ceac02 is 175, key is test_row_0/A:col10/1732648470335/Put/seqid=0 2024-11-26T19:14:31,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742033_1209 (size=31255) 2024-11-26T19:14:31,779 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=271, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/A/fbafe2f643924acca20660e5b3ceac02 2024-11-26T19:14:31,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/B/0759c100aa7845578cd0ae42bfe7de2d is 50, key is test_row_0/B:col10/1732648470335/Put/seqid=0 2024-11-26T19:14:31,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742034_1210 (size=12301) 2024-11-26T19:14:31,862 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:31,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39332 deadline: 1732648531862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:31,863 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:31,863 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:31,863 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:32,177 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:32,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39332 deadline: 1732648532172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:32,178 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:32,178 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:32,178 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:32,199 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/B/0759c100aa7845578cd0ae42bfe7de2d 2024-11-26T19:14:32,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/C/51524173443d410f9c7e68ae1961096a is 50, key is test_row_0/C:col10/1732648470335/Put/seqid=0 2024-11-26T19:14:32,212 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:32,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39324 deadline: 1732648532212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:32,213 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to fe76acf732b2,42785,1732648423970 for 'test_row_2' in d23b79ada6d885c09c429afcb7484518 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8202 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T19:14:32,213 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:32,213 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:32,214 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:32,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742035_1211 (size=12301) 2024-11-26T19:14:32,237 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:32,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39350 deadline: 1732648532232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:32,238 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to fe76acf732b2,42785,1732648423970 for 'test_row_0' in d23b79ada6d885c09c429afcb7484518 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8224 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T19:14:32,238 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:32,238 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:32,238 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:32,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-26T19:14:32,619 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/C/51524173443d410f9c7e68ae1961096a 2024-11-26T19:14:32,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/A/fbafe2f643924acca20660e5b3ceac02 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/fbafe2f643924acca20660e5b3ceac02 2024-11-26T19:14:32,630 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/fbafe2f643924acca20660e5b3ceac02, entries=150, sequenceid=271, filesize=30.5 K 2024-11-26T19:14:32,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/B/0759c100aa7845578cd0ae42bfe7de2d as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/0759c100aa7845578cd0ae42bfe7de2d 2024-11-26T19:14:32,638 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/0759c100aa7845578cd0ae42bfe7de2d, entries=150, sequenceid=271, filesize=12.0 K 2024-11-26T19:14:32,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/C/51524173443d410f9c7e68ae1961096a as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/51524173443d410f9c7e68ae1961096a 2024-11-26T19:14:32,654 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/51524173443d410f9c7e68ae1961096a, entries=150, sequenceid=271, filesize=12.0 K 2024-11-26T19:14:32,658 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for d23b79ada6d885c09c429afcb7484518 in 1316ms, sequenceid=271, compaction requested=false 2024-11-26T19:14:32,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for d23b79ada6d885c09c429afcb7484518: 2024-11-26T19:14:32,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. 2024-11-26T19:14:32,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-11-26T19:14:32,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-11-26T19:14:32,663 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-26T19:14:32,663 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2450 sec 2024-11-26T19:14:32,665 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 2.2510 sec 2024-11-26T19:14:32,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(8855): Flush requested on d23b79ada6d885c09c429afcb7484518 2024-11-26T19:14:32,689 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d23b79ada6d885c09c429afcb7484518 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-26T19:14:32,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d23b79ada6d885c09c429afcb7484518, store=A 2024-11-26T19:14:32,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:14:32,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d23b79ada6d885c09c429afcb7484518, store=B 2024-11-26T19:14:32,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:14:32,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d23b79ada6d885c09c429afcb7484518, store=C 2024-11-26T19:14:32,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:14:32,704 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126e1e0e6beeea34ff19f545205665625ad_d23b79ada6d885c09c429afcb7484518 is 50, key is test_row_0/A:col10/1732648472685/Put/seqid=0 2024-11-26T19:14:32,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742036_1212 (size=12454) 2024-11-26T19:14:32,772 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:32,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39332 deadline: 1732648532764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:32,774 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:32,774 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:32,774 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:32,886 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:32,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39332 deadline: 1732648532882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:32,887 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:32,887 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:32,887 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:33,098 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:33,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39332 deadline: 1732648533092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:33,099 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:33,099 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:33,099 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:33,125 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:33,131 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126e1e0e6beeea34ff19f545205665625ad_d23b79ada6d885c09c429afcb7484518 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126e1e0e6beeea34ff19f545205665625ad_d23b79ada6d885c09c429afcb7484518 2024-11-26T19:14:33,132 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/A/7e33ffdee4e7427e86868e6b19b30ff4, store: [table=TestAcidGuarantees family=A region=d23b79ada6d885c09c429afcb7484518] 2024-11-26T19:14:33,133 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/A/7e33ffdee4e7427e86868e6b19b30ff4 is 175, key is test_row_0/A:col10/1732648472685/Put/seqid=0 2024-11-26T19:14:33,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742037_1213 (size=31255) 2024-11-26T19:14:33,178 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=288, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/A/7e33ffdee4e7427e86868e6b19b30ff4 2024-11-26T19:14:33,212 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/B/e28e2c5d76ae4c0593cb903a0ec2075a is 50, key is test_row_0/B:col10/1732648472685/Put/seqid=0 2024-11-26T19:14:33,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742038_1214 (size=12301) 2024-11-26T19:14:33,404 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:33,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39332 deadline: 1732648533401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:33,407 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:33,407 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:33,407 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:33,576 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:33,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39306 deadline: 1732648533572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:33,577 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to fe76acf732b2,42785,1732648423970 for 'test_row_0' in d23b79ada6d885c09c429afcb7484518 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4175 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T19:14:33,578 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:33,578 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:33,578 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:33,606 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:33,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39292 deadline: 1732648533602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:33,607 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to fe76acf732b2,42785,1732648423970 for 'test_row_2' in d23b79ada6d885c09c429afcb7484518 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4203 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T19:14:33,607 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:33,608 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:33,608 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:33,636 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/B/e28e2c5d76ae4c0593cb903a0ec2075a 2024-11-26T19:14:33,656 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/C/4b1500fa97fb40d591b01f2a73ceed34 is 50, key is test_row_0/C:col10/1732648472685/Put/seqid=0 2024-11-26T19:14:33,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742039_1215 (size=12301) 2024-11-26T19:14:33,703 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/C/4b1500fa97fb40d591b01f2a73ceed34 2024-11-26T19:14:33,710 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/A/7e33ffdee4e7427e86868e6b19b30ff4 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/7e33ffdee4e7427e86868e6b19b30ff4 2024-11-26T19:14:33,717 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/7e33ffdee4e7427e86868e6b19b30ff4, entries=150, sequenceid=288, filesize=30.5 K 2024-11-26T19:14:33,719 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/B/e28e2c5d76ae4c0593cb903a0ec2075a as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/e28e2c5d76ae4c0593cb903a0ec2075a 2024-11-26T19:14:33,725 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/e28e2c5d76ae4c0593cb903a0ec2075a, entries=150, sequenceid=288, filesize=12.0 K 2024-11-26T19:14:33,727 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/C/4b1500fa97fb40d591b01f2a73ceed34 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/4b1500fa97fb40d591b01f2a73ceed34 2024-11-26T19:14:33,733 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/4b1500fa97fb40d591b01f2a73ceed34, entries=150, sequenceid=288, filesize=12.0 K 2024-11-26T19:14:33,736 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for d23b79ada6d885c09c429afcb7484518 in 1046ms, sequenceid=288, compaction requested=true 2024-11-26T19:14:33,736 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d23b79ada6d885c09c429afcb7484518: 2024-11-26T19:14:33,736 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d23b79ada6d885c09c429afcb7484518:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T19:14:33,736 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:14:33,736 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T19:14:33,736 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T19:14:33,736 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d23b79ada6d885c09c429afcb7484518:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T19:14:33,736 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:14:33,736 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d23b79ada6d885c09c429afcb7484518:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T19:14:33,736 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T19:14:33,737 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37367 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T19:14:33,737 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1541): d23b79ada6d885c09c429afcb7484518/B is initiating minor compaction (all files) 2024-11-26T19:14:33,738 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d23b79ada6d885c09c429afcb7484518/B in TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. 2024-11-26T19:14:33,738 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/96e1560c0b164ff5bcc212fd25486acf, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/0759c100aa7845578cd0ae42bfe7de2d, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/e28e2c5d76ae4c0593cb903a0ec2075a] into tmpdir=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp, totalSize=36.5 K 2024-11-26T19:14:33,738 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94229 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T19:14:33,738 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1541): d23b79ada6d885c09c429afcb7484518/A is initiating minor compaction (all files) 2024-11-26T19:14:33,738 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d23b79ada6d885c09c429afcb7484518/A in TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. 2024-11-26T19:14:33,738 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/8f1d34e65e1947c49f150925f1a3cd26, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/fbafe2f643924acca20660e5b3ceac02, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/7e33ffdee4e7427e86868e6b19b30ff4] into tmpdir=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp, totalSize=92.0 K 2024-11-26T19:14:33,738 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. 2024-11-26T19:14:33,738 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. files: [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/8f1d34e65e1947c49f150925f1a3cd26, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/fbafe2f643924acca20660e5b3ceac02, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/7e33ffdee4e7427e86868e6b19b30ff4] 2024-11-26T19:14:33,740 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting 96e1560c0b164ff5bcc212fd25486acf, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1732648469390 2024-11-26T19:14:33,740 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8f1d34e65e1947c49f150925f1a3cd26, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1732648469390 2024-11-26T19:14:33,740 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting fbafe2f643924acca20660e5b3ceac02, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1732648470313 2024-11-26T19:14:33,740 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting 0759c100aa7845578cd0ae42bfe7de2d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1732648470313 2024-11-26T19:14:33,741 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7e33ffdee4e7427e86868e6b19b30ff4, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1732648471530 2024-11-26T19:14:33,741 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting e28e2c5d76ae4c0593cb903a0ec2075a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1732648471530 2024-11-26T19:14:33,759 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d23b79ada6d885c09c429afcb7484518] 2024-11-26T19:14:33,766 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d23b79ada6d885c09c429afcb7484518#B#compaction#180 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T19:14:33,767 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/B/5e159984f99c41a8b4f1885cbb877ee2 is 50, key is test_row_0/B:col10/1732648472685/Put/seqid=0 2024-11-26T19:14:33,769 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411267781aa2b0d004bf78980be7425845820_d23b79ada6d885c09c429afcb7484518 store=[table=TestAcidGuarantees family=A region=d23b79ada6d885c09c429afcb7484518] 2024-11-26T19:14:33,770 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411267781aa2b0d004bf78980be7425845820_d23b79ada6d885c09c429afcb7484518, store=[table=TestAcidGuarantees family=A region=d23b79ada6d885c09c429afcb7484518] 2024-11-26T19:14:33,770 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411267781aa2b0d004bf78980be7425845820_d23b79ada6d885c09c429afcb7484518 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d23b79ada6d885c09c429afcb7484518] 2024-11-26T19:14:33,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742040_1216 (size=13017) 2024-11-26T19:14:33,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742041_1217 (size=4469) 2024-11-26T19:14:33,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(8855): Flush requested on d23b79ada6d885c09c429afcb7484518 2024-11-26T19:14:33,916 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d23b79ada6d885c09c429afcb7484518 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-26T19:14:33,916 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d23b79ada6d885c09c429afcb7484518, store=A 2024-11-26T19:14:33,916 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:14:33,916 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d23b79ada6d885c09c429afcb7484518, store=B 2024-11-26T19:14:33,916 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:14:33,916 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d23b79ada6d885c09c429afcb7484518, store=C 2024-11-26T19:14:33,916 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:14:33,935 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126704cc4aefb3743abb8bb4de6b79b480d_d23b79ada6d885c09c429afcb7484518 is 50, key is test_row_0/A:col10/1732648473914/Put/seqid=0 2024-11-26T19:14:33,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742042_1218 (size=12454) 2024-11-26T19:14:33,959 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:33,966 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126704cc4aefb3743abb8bb4de6b79b480d_d23b79ada6d885c09c429afcb7484518 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126704cc4aefb3743abb8bb4de6b79b480d_d23b79ada6d885c09c429afcb7484518 2024-11-26T19:14:33,967 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/A/8d976e9543b347658e3260e323c8fba0, store: [table=TestAcidGuarantees family=A region=d23b79ada6d885c09c429afcb7484518] 2024-11-26T19:14:33,967 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/A/8d976e9543b347658e3260e323c8fba0 is 175, key is test_row_0/A:col10/1732648473914/Put/seqid=0 2024-11-26T19:14:33,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742043_1219 (size=31255) 2024-11-26T19:14:33,984 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:33,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39332 deadline: 1732648533977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:33,984 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:34,002 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:34,002 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:34,112 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:34,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39332 deadline: 1732648534111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:34,113 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:34,113 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:34,113 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:34,192 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/B/5e159984f99c41a8b4f1885cbb877ee2 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/5e159984f99c41a8b4f1885cbb877ee2 2024-11-26T19:14:34,196 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d23b79ada6d885c09c429afcb7484518#A#compaction#179 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T19:14:34,197 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/A/e8311f3d5af74e5e81597a7261fb4780 is 175, key is test_row_0/A:col10/1732648472685/Put/seqid=0 2024-11-26T19:14:34,199 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d23b79ada6d885c09c429afcb7484518/B of d23b79ada6d885c09c429afcb7484518 into 5e159984f99c41a8b4f1885cbb877ee2(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T19:14:34,200 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d23b79ada6d885c09c429afcb7484518: 2024-11-26T19:14:34,200 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., storeName=d23b79ada6d885c09c429afcb7484518/B, priority=13, startTime=1732648473736; duration=0sec 2024-11-26T19:14:34,200 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T19:14:34,200 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d23b79ada6d885c09c429afcb7484518:B 2024-11-26T19:14:34,200 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T19:14:34,201 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37367 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T19:14:34,201 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1541): d23b79ada6d885c09c429afcb7484518/C is initiating minor compaction (all files) 2024-11-26T19:14:34,202 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d23b79ada6d885c09c429afcb7484518/C in TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. 2024-11-26T19:14:34,202 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/a97470276177497c951a479e55ec275a, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/51524173443d410f9c7e68ae1961096a, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/4b1500fa97fb40d591b01f2a73ceed34] into tmpdir=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp, totalSize=36.5 K 2024-11-26T19:14:34,202 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting a97470276177497c951a479e55ec275a, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1732648469390 2024-11-26T19:14:34,203 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting 51524173443d410f9c7e68ae1961096a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1732648470313 2024-11-26T19:14:34,203 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting 4b1500fa97fb40d591b01f2a73ceed34, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1732648471530 2024-11-26T19:14:34,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742044_1220 (size=31971) 2024-11-26T19:14:34,222 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d23b79ada6d885c09c429afcb7484518#C#compaction#182 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T19:14:34,223 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/C/52d6badf775648aba87302a9eb5c2b6a is 50, key is test_row_0/C:col10/1732648472685/Put/seqid=0 2024-11-26T19:14:34,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742045_1221 (size=13017) 2024-11-26T19:14:34,229 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/A/e8311f3d5af74e5e81597a7261fb4780 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/e8311f3d5af74e5e81597a7261fb4780 2024-11-26T19:14:34,234 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/C/52d6badf775648aba87302a9eb5c2b6a as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/52d6badf775648aba87302a9eb5c2b6a 2024-11-26T19:14:34,235 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d23b79ada6d885c09c429afcb7484518/A of d23b79ada6d885c09c429afcb7484518 into e8311f3d5af74e5e81597a7261fb4780(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T19:14:34,235 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d23b79ada6d885c09c429afcb7484518: 2024-11-26T19:14:34,235 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., storeName=d23b79ada6d885c09c429afcb7484518/A, priority=13, startTime=1732648473736; duration=0sec 2024-11-26T19:14:34,235 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:14:34,235 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d23b79ada6d885c09c429afcb7484518:A 2024-11-26T19:14:34,243 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d23b79ada6d885c09c429afcb7484518/C of d23b79ada6d885c09c429afcb7484518 into 52d6badf775648aba87302a9eb5c2b6a(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T19:14:34,243 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d23b79ada6d885c09c429afcb7484518: 2024-11-26T19:14:34,243 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., storeName=d23b79ada6d885c09c429afcb7484518/C, priority=13, startTime=1732648473736; duration=0sec 2024-11-26T19:14:34,243 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:14:34,243 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d23b79ada6d885c09c429afcb7484518:C 2024-11-26T19:14:34,324 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:34,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39332 deadline: 1732648534322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:34,325 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:34,325 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:34,325 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:34,378 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=306, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/A/8d976e9543b347658e3260e323c8fba0 2024-11-26T19:14:34,389 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/B/6d0349a759884b6b8e57d6420b7594cf is 50, key is test_row_0/B:col10/1732648473914/Put/seqid=0 2024-11-26T19:14:34,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742046_1222 (size=12301) 2024-11-26T19:14:34,400 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=306 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/B/6d0349a759884b6b8e57d6420b7594cf 2024-11-26T19:14:34,410 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/C/c80bc1340b25432cbbb5d53334c161d9 is 50, key is test_row_0/C:col10/1732648473914/Put/seqid=0 2024-11-26T19:14:34,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742047_1223 (size=12301) 2024-11-26T19:14:34,419 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=306 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/C/c80bc1340b25432cbbb5d53334c161d9 2024-11-26T19:14:34,425 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/A/8d976e9543b347658e3260e323c8fba0 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/8d976e9543b347658e3260e323c8fba0 2024-11-26T19:14:34,429 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/8d976e9543b347658e3260e323c8fba0, entries=150, sequenceid=306, filesize=30.5 K 2024-11-26T19:14:34,431 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/B/6d0349a759884b6b8e57d6420b7594cf as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/6d0349a759884b6b8e57d6420b7594cf 2024-11-26T19:14:34,436 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/6d0349a759884b6b8e57d6420b7594cf, entries=150, sequenceid=306, filesize=12.0 K 2024-11-26T19:14:34,437 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/C/c80bc1340b25432cbbb5d53334c161d9 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/c80bc1340b25432cbbb5d53334c161d9 2024-11-26T19:14:34,443 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/c80bc1340b25432cbbb5d53334c161d9, entries=150, sequenceid=306, filesize=12.0 K 2024-11-26T19:14:34,443 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for d23b79ada6d885c09c429afcb7484518 in 527ms, sequenceid=306, compaction requested=false 2024-11-26T19:14:34,444 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d23b79ada6d885c09c429afcb7484518: 2024-11-26T19:14:34,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-26T19:14:34,582 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T19:14:34,583 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T19:14:34,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-11-26T19:14:34,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-26T19:14:34,585 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T19:14:34,586 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T19:14:34,587 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T19:14:34,634 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d23b79ada6d885c09c429afcb7484518 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-26T19:14:34,635 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d23b79ada6d885c09c429afcb7484518, store=A 2024-11-26T19:14:34,635 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:14:34,635 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d23b79ada6d885c09c429afcb7484518, store=B 2024-11-26T19:14:34,635 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:14:34,635 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d23b79ada6d885c09c429afcb7484518, store=C 2024-11-26T19:14:34,635 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:14:34,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(8855): Flush requested on d23b79ada6d885c09c429afcb7484518 2024-11-26T19:14:34,671 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126971e46746155449d8509f206585fb225_d23b79ada6d885c09c429afcb7484518 is 50, key is test_row_0/A:col10/1732648473942/Put/seqid=0 2024-11-26T19:14:34,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-26T19:14:34,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742048_1224 (size=14994) 2024-11-26T19:14:34,729 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:34,734 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126971e46746155449d8509f206585fb225_d23b79ada6d885c09c429afcb7484518 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126971e46746155449d8509f206585fb225_d23b79ada6d885c09c429afcb7484518 2024-11-26T19:14:34,735 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/A/048a166dcbec4c2c9cf2bdbea89fed41, store: [table=TestAcidGuarantees family=A region=d23b79ada6d885c09c429afcb7484518] 2024-11-26T19:14:34,736 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/A/048a166dcbec4c2c9cf2bdbea89fed41 is 175, key is test_row_0/A:col10/1732648473942/Put/seqid=0 2024-11-26T19:14:34,739 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-26T19:14:34,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. 2024-11-26T19:14:34,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. as already flushing 2024-11-26T19:14:34,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. 2024-11-26T19:14:34,739 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => d23b79ada6d885c09c429afcb7484518, NAME => 'TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:34,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => d23b79ada6d885c09c429afcb7484518, NAME => 'TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:34,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d23b79ada6d885c09c429afcb7484518, NAME => 'TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d23b79ada6d885c09c429afcb7484518, NAME => 'TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:34,747 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:34,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39332 deadline: 1732648534743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:34,747 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:34,747 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:34,747 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:34,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742049_1225 (size=39949) 2024-11-26T19:14:34,786 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=324, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/A/048a166dcbec4c2c9cf2bdbea89fed41 2024-11-26T19:14:34,811 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/B/47f399bfc1b84f7d8e5f55c5073fbec3 is 50, key is test_row_0/B:col10/1732648473942/Put/seqid=0 2024-11-26T19:14:34,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742050_1226 (size=12301) 2024-11-26T19:14:34,855 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:34,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39332 deadline: 1732648534852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:34,856 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:34,856 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:34,856 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:34,893 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-26T19:14:34,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. 2024-11-26T19:14:34,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. as already flushing 2024-11-26T19:14:34,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. 2024-11-26T19:14:34,893 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => d23b79ada6d885c09c429afcb7484518, NAME => 'TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:34,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => d23b79ada6d885c09c429afcb7484518, NAME => 'TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:34,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d23b79ada6d885c09c429afcb7484518, NAME => 'TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d23b79ada6d885c09c429afcb7484518, NAME => 'TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:34,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-26T19:14:35,047 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-26T19:14:35,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. 2024-11-26T19:14:35,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. as already flushing 2024-11-26T19:14:35,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. 2024-11-26T19:14:35,050 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => d23b79ada6d885c09c429afcb7484518, NAME => 'TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:35,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => d23b79ada6d885c09c429afcb7484518, NAME => 'TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:35,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d23b79ada6d885c09c429afcb7484518, NAME => 'TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d23b79ada6d885c09c429afcb7484518, NAME => 'TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:35,062 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:35,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39332 deadline: 1732648535062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:35,064 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:35,064 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:35,064 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:35,204 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-26T19:14:35,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. 2024-11-26T19:14:35,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. as already flushing 2024-11-26T19:14:35,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. 2024-11-26T19:14:35,204 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => d23b79ada6d885c09c429afcb7484518, NAME => 'TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:35,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => d23b79ada6d885c09c429afcb7484518, NAME => 'TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:35,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d23b79ada6d885c09c429afcb7484518, NAME => 'TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d23b79ada6d885c09c429afcb7484518, NAME => 'TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:35,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-26T19:14:35,247 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/B/47f399bfc1b84f7d8e5f55c5073fbec3 2024-11-26T19:14:35,260 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/C/c238a51298e445a7bd70307b32e97eb5 is 50, key is test_row_0/C:col10/1732648473942/Put/seqid=0 2024-11-26T19:14:35,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742051_1227 (size=12301) 2024-11-26T19:14:35,282 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/C/c238a51298e445a7bd70307b32e97eb5 2024-11-26T19:14:35,288 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/A/048a166dcbec4c2c9cf2bdbea89fed41 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/048a166dcbec4c2c9cf2bdbea89fed41 2024-11-26T19:14:35,293 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/048a166dcbec4c2c9cf2bdbea89fed41, entries=200, sequenceid=324, filesize=39.0 K 2024-11-26T19:14:35,294 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/B/47f399bfc1b84f7d8e5f55c5073fbec3 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/47f399bfc1b84f7d8e5f55c5073fbec3 2024-11-26T19:14:35,299 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/47f399bfc1b84f7d8e5f55c5073fbec3, entries=150, sequenceid=324, filesize=12.0 K 2024-11-26T19:14:35,300 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/C/c238a51298e445a7bd70307b32e97eb5 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/c238a51298e445a7bd70307b32e97eb5 2024-11-26T19:14:35,315 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/c238a51298e445a7bd70307b32e97eb5, entries=150, sequenceid=324, filesize=12.0 K 2024-11-26T19:14:35,318 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for d23b79ada6d885c09c429afcb7484518 in 684ms, sequenceid=324, compaction requested=true 2024-11-26T19:14:35,318 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d23b79ada6d885c09c429afcb7484518: 2024-11-26T19:14:35,318 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d23b79ada6d885c09c429afcb7484518:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T19:14:35,318 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T19:14:35,319 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:14:35,319 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T19:14:35,319 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d23b79ada6d885c09c429afcb7484518:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T19:14:35,319 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:14:35,319 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d23b79ada6d885c09c429afcb7484518:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T19:14:35,319 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T19:14:35,320 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103175 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T19:14:35,320 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1541): d23b79ada6d885c09c429afcb7484518/A is initiating minor compaction (all files) 2024-11-26T19:14:35,320 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d23b79ada6d885c09c429afcb7484518/A in TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. 2024-11-26T19:14:35,320 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/e8311f3d5af74e5e81597a7261fb4780, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/8d976e9543b347658e3260e323c8fba0, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/048a166dcbec4c2c9cf2bdbea89fed41] into tmpdir=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp, totalSize=100.8 K 2024-11-26T19:14:35,320 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. 2024-11-26T19:14:35,320 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. files: [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/e8311f3d5af74e5e81597a7261fb4780, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/8d976e9543b347658e3260e323c8fba0, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/048a166dcbec4c2c9cf2bdbea89fed41] 2024-11-26T19:14:35,321 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T19:14:35,321 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1541): d23b79ada6d885c09c429afcb7484518/B is initiating minor compaction (all files) 2024-11-26T19:14:35,321 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d23b79ada6d885c09c429afcb7484518/B in TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. 2024-11-26T19:14:35,321 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/5e159984f99c41a8b4f1885cbb877ee2, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/6d0349a759884b6b8e57d6420b7594cf, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/47f399bfc1b84f7d8e5f55c5073fbec3] into tmpdir=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp, totalSize=36.7 K 2024-11-26T19:14:35,322 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting e8311f3d5af74e5e81597a7261fb4780, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1732648471530 2024-11-26T19:14:35,324 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8d976e9543b347658e3260e323c8fba0, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1732648472723 2024-11-26T19:14:35,324 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting 5e159984f99c41a8b4f1885cbb877ee2, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1732648471530 2024-11-26T19:14:35,324 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting 048a166dcbec4c2c9cf2bdbea89fed41, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1732648473942 2024-11-26T19:14:35,324 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting 6d0349a759884b6b8e57d6420b7594cf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1732648472723 2024-11-26T19:14:35,325 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting 47f399bfc1b84f7d8e5f55c5073fbec3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1732648473942 2024-11-26T19:14:35,337 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d23b79ada6d885c09c429afcb7484518] 2024-11-26T19:14:35,338 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d23b79ada6d885c09c429afcb7484518#B#compaction#188 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T19:14:35,339 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/B/290007cee1454f9fa36fa5a9062f90d9 is 50, key is test_row_0/B:col10/1732648473942/Put/seqid=0 2024-11-26T19:14:35,343 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241126402510a6b1f9467da4d8b75c0df0594d_d23b79ada6d885c09c429afcb7484518 store=[table=TestAcidGuarantees family=A region=d23b79ada6d885c09c429afcb7484518] 2024-11-26T19:14:35,345 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241126402510a6b1f9467da4d8b75c0df0594d_d23b79ada6d885c09c429afcb7484518, store=[table=TestAcidGuarantees family=A region=d23b79ada6d885c09c429afcb7484518] 2024-11-26T19:14:35,345 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126402510a6b1f9467da4d8b75c0df0594d_d23b79ada6d885c09c429afcb7484518 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d23b79ada6d885c09c429afcb7484518] 2024-11-26T19:14:35,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742052_1228 (size=13119) 2024-11-26T19:14:35,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742053_1229 (size=4469) 2024-11-26T19:14:35,358 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-26T19:14:35,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. 2024-11-26T19:14:35,358 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing d23b79ada6d885c09c429afcb7484518 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-26T19:14:35,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d23b79ada6d885c09c429afcb7484518, store=A 2024-11-26T19:14:35,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:14:35,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d23b79ada6d885c09c429afcb7484518, store=B 2024-11-26T19:14:35,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:14:35,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d23b79ada6d885c09c429afcb7484518, store=C 2024-11-26T19:14:35,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:14:35,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126c994d2954d5e412894a8303b6a815858_d23b79ada6d885c09c429afcb7484518 is 50, key is test_row_0/A:col10/1732648474733/Put/seqid=0 2024-11-26T19:14:35,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(8855): Flush requested on d23b79ada6d885c09c429afcb7484518 2024-11-26T19:14:35,375 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. as already flushing 2024-11-26T19:14:35,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742054_1230 (size=12454) 2024-11-26T19:14:35,447 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:35,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39332 deadline: 1732648535441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:35,450 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:35,450 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:35,450 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:35,557 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:35,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39332 deadline: 1732648535551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:35,558 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:35,558 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:35,558 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:35,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-26T19:14:35,759 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/B/290007cee1454f9fa36fa5a9062f90d9 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/290007cee1454f9fa36fa5a9062f90d9 2024-11-26T19:14:35,760 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d23b79ada6d885c09c429afcb7484518#A#compaction#189 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T19:14:35,760 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/A/dfc123e1f18a49ef978a840f82eac11f is 175, key is test_row_0/A:col10/1732648473942/Put/seqid=0 2024-11-26T19:14:35,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742055_1231 (size=32073) 2024-11-26T19:14:35,769 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:35,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39332 deadline: 1732648535762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:35,769 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:35,769 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:35,769 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:35,769 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d23b79ada6d885c09c429afcb7484518/B of d23b79ada6d885c09c429afcb7484518 into 290007cee1454f9fa36fa5a9062f90d9(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T19:14:35,769 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d23b79ada6d885c09c429afcb7484518: 2024-11-26T19:14:35,769 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., storeName=d23b79ada6d885c09c429afcb7484518/B, priority=13, startTime=1732648475319; duration=0sec 2024-11-26T19:14:35,770 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T19:14:35,770 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d23b79ada6d885c09c429afcb7484518:B 2024-11-26T19:14:35,770 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T19:14:35,778 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T19:14:35,778 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1541): d23b79ada6d885c09c429afcb7484518/C is initiating minor compaction (all files) 2024-11-26T19:14:35,778 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d23b79ada6d885c09c429afcb7484518/C in TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. 2024-11-26T19:14:35,778 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/52d6badf775648aba87302a9eb5c2b6a, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/c80bc1340b25432cbbb5d53334c161d9, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/c238a51298e445a7bd70307b32e97eb5] into tmpdir=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp, totalSize=36.7 K 2024-11-26T19:14:35,779 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting 52d6badf775648aba87302a9eb5c2b6a, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1732648471530 2024-11-26T19:14:35,779 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting c80bc1340b25432cbbb5d53334c161d9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1732648472723 2024-11-26T19:14:35,779 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting c238a51298e445a7bd70307b32e97eb5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1732648473942 2024-11-26T19:14:35,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:35,787 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d23b79ada6d885c09c429afcb7484518#C#compaction#191 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T19:14:35,788 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/C/9ac2ea240ffd488c9952c3579be4e097 is 50, key is test_row_0/C:col10/1732648473942/Put/seqid=0 2024-11-26T19:14:35,789 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126c994d2954d5e412894a8303b6a815858_d23b79ada6d885c09c429afcb7484518 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126c994d2954d5e412894a8303b6a815858_d23b79ada6d885c09c429afcb7484518 2024-11-26T19:14:35,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/A/28618b3b3c864dcb802465d4acebf520, store: [table=TestAcidGuarantees family=A region=d23b79ada6d885c09c429afcb7484518] 2024-11-26T19:14:35,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/A/28618b3b3c864dcb802465d4acebf520 is 175, key is test_row_0/A:col10/1732648474733/Put/seqid=0 2024-11-26T19:14:35,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742056_1232 (size=13119) 2024-11-26T19:14:35,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742057_1233 (size=31255) 2024-11-26T19:14:35,813 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/C/9ac2ea240ffd488c9952c3579be4e097 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/9ac2ea240ffd488c9952c3579be4e097 2024-11-26T19:14:35,819 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d23b79ada6d885c09c429afcb7484518/C of d23b79ada6d885c09c429afcb7484518 into 9ac2ea240ffd488c9952c3579be4e097(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T19:14:35,819 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d23b79ada6d885c09c429afcb7484518: 2024-11-26T19:14:35,819 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., storeName=d23b79ada6d885c09c429afcb7484518/C, priority=13, startTime=1732648475319; duration=0sec 2024-11-26T19:14:35,819 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:14:35,819 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d23b79ada6d885c09c429afcb7484518:C 2024-11-26T19:14:36,085 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:36,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39332 deadline: 1732648536082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:36,086 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:36,086 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:36,086 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:36,172 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/A/dfc123e1f18a49ef978a840f82eac11f as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/dfc123e1f18a49ef978a840f82eac11f 2024-11-26T19:14:36,177 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d23b79ada6d885c09c429afcb7484518/A of d23b79ada6d885c09c429afcb7484518 into dfc123e1f18a49ef978a840f82eac11f(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T19:14:36,177 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d23b79ada6d885c09c429afcb7484518: 2024-11-26T19:14:36,177 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., storeName=d23b79ada6d885c09c429afcb7484518/A, priority=13, startTime=1732648475318; duration=0sec 2024-11-26T19:14:36,178 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:14:36,178 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d23b79ada6d885c09c429afcb7484518:A 2024-11-26T19:14:36,208 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=341, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/A/28618b3b3c864dcb802465d4acebf520 2024-11-26T19:14:36,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/B/1016ccddb6d94ca4aaa8f83292074fd3 is 50, key is test_row_0/B:col10/1732648474733/Put/seqid=0 2024-11-26T19:14:36,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742058_1234 (size=12301) 2024-11-26T19:14:36,592 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:36,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39332 deadline: 1732648536591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:36,593 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:36,593 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:36,593 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:36,644 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=341 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/B/1016ccddb6d94ca4aaa8f83292074fd3 2024-11-26T19:14:36,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/C/fcc5e451cdc04b12afd59db7f94992b2 is 50, key is test_row_0/C:col10/1732648474733/Put/seqid=0 2024-11-26T19:14:36,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742059_1235 (size=12301) 2024-11-26T19:14:36,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-26T19:14:37,102 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=341 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/C/fcc5e451cdc04b12afd59db7f94992b2 2024-11-26T19:14:37,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/A/28618b3b3c864dcb802465d4acebf520 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/28618b3b3c864dcb802465d4acebf520 2024-11-26T19:14:37,119 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/28618b3b3c864dcb802465d4acebf520, entries=150, sequenceid=341, filesize=30.5 K 2024-11-26T19:14:37,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/B/1016ccddb6d94ca4aaa8f83292074fd3 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/1016ccddb6d94ca4aaa8f83292074fd3 2024-11-26T19:14:37,129 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/1016ccddb6d94ca4aaa8f83292074fd3, entries=150, sequenceid=341, filesize=12.0 K 2024-11-26T19:14:37,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/C/fcc5e451cdc04b12afd59db7f94992b2 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/fcc5e451cdc04b12afd59db7f94992b2 2024-11-26T19:14:37,143 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/fcc5e451cdc04b12afd59db7f94992b2, entries=150, sequenceid=341, filesize=12.0 K 2024-11-26T19:14:37,145 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for d23b79ada6d885c09c429afcb7484518 in 1787ms, sequenceid=341, compaction requested=false 2024-11-26T19:14:37,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for d23b79ada6d885c09c429afcb7484518: 2024-11-26T19:14:37,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. 2024-11-26T19:14:37,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-11-26T19:14:37,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-11-26T19:14:37,155 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-26T19:14:37,155 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5620 sec 2024-11-26T19:14:37,166 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 2.5730 sec 2024-11-26T19:14:37,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(8855): Flush requested on d23b79ada6d885c09c429afcb7484518 2024-11-26T19:14:37,608 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d23b79ada6d885c09c429afcb7484518 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-26T19:14:37,610 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d23b79ada6d885c09c429afcb7484518, store=A 2024-11-26T19:14:37,610 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:14:37,610 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d23b79ada6d885c09c429afcb7484518, store=B 2024-11-26T19:14:37,610 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:14:37,610 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d23b79ada6d885c09c429afcb7484518, store=C 2024-11-26T19:14:37,610 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:14:37,638 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126093bea2423844ad188d810ca49f86b49_d23b79ada6d885c09c429afcb7484518 is 50, key is test_row_0/A:col10/1732648477606/Put/seqid=0 2024-11-26T19:14:37,644 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:37,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39306 deadline: 1732648537637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:37,644 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:37,644 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:37,644 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:37,646 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:37,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39292 deadline: 1732648537644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:37,646 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:37,646 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:37,646 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:37,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742060_1236 (size=14994) 2024-11-26T19:14:37,654 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:37,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39332 deadline: 1732648537647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:37,655 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:37,656 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:37,656 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:37,656 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:37,661 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126093bea2423844ad188d810ca49f86b49_d23b79ada6d885c09c429afcb7484518 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126093bea2423844ad188d810ca49f86b49_d23b79ada6d885c09c429afcb7484518 2024-11-26T19:14:37,662 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/A/4e49b55a13c5477193e5572460b62115, store: [table=TestAcidGuarantees family=A region=d23b79ada6d885c09c429afcb7484518] 2024-11-26T19:14:37,663 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/A/4e49b55a13c5477193e5572460b62115 is 175, key is test_row_0/A:col10/1732648477606/Put/seqid=0 2024-11-26T19:14:37,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742061_1237 (size=39949) 2024-11-26T19:14:37,758 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:37,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39306 deadline: 1732648537752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:37,758 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:37,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39292 deadline: 1732648537752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:37,758 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:37,758 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:37,758 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:37,759 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:37,759 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:37,759 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:37,765 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:37,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39332 deadline: 1732648537762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:37,765 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:37,765 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:37,765 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:37,965 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:37,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39306 deadline: 1732648537962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:37,965 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:37,965 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:37,965 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:37,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39292 deadline: 1732648537962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:37,965 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:37,966 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:37,966 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:37,966 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:37,972 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:37,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39332 deadline: 1732648537971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:37,972 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:37,972 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:37,972 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:38,071 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=361, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/A/4e49b55a13c5477193e5572460b62115 2024-11-26T19:14:38,080 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/B/5173fca9e04d469e9814c5043a47b8ed is 50, key is test_row_0/B:col10/1732648477606/Put/seqid=0 2024-11-26T19:14:38,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742062_1238 (size=12301) 2024-11-26T19:14:38,277 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:38,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39306 deadline: 1732648538272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:38,277 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:38,277 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:38,277 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:38,277 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:38,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39292 deadline: 1732648538272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:38,278 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:38,278 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:38,278 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:38,283 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:38,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39332 deadline: 1732648538282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:38,284 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:38,284 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:38,284 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:38,485 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=361 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/B/5173fca9e04d469e9814c5043a47b8ed 2024-11-26T19:14:38,507 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/C/60787e1bebea4a948f6a993bbb262299 is 50, key is test_row_0/C:col10/1732648477606/Put/seqid=0 2024-11-26T19:14:38,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742063_1239 (size=12301) 2024-11-26T19:14:38,531 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=361 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/C/60787e1bebea4a948f6a993bbb262299 2024-11-26T19:14:38,537 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/A/4e49b55a13c5477193e5572460b62115 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/4e49b55a13c5477193e5572460b62115 2024-11-26T19:14:38,541 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/4e49b55a13c5477193e5572460b62115, entries=200, sequenceid=361, filesize=39.0 K 2024-11-26T19:14:38,543 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/B/5173fca9e04d469e9814c5043a47b8ed as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/5173fca9e04d469e9814c5043a47b8ed 2024-11-26T19:14:38,557 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/5173fca9e04d469e9814c5043a47b8ed, entries=150, sequenceid=361, filesize=12.0 K 2024-11-26T19:14:38,558 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/C/60787e1bebea4a948f6a993bbb262299 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/60787e1bebea4a948f6a993bbb262299 2024-11-26T19:14:38,562 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/60787e1bebea4a948f6a993bbb262299, entries=150, sequenceid=361, filesize=12.0 K 2024-11-26T19:14:38,564 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=87.22 KB/89310 for d23b79ada6d885c09c429afcb7484518 in 956ms, sequenceid=361, compaction requested=true 2024-11-26T19:14:38,564 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d23b79ada6d885c09c429afcb7484518: 2024-11-26T19:14:38,564 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d23b79ada6d885c09c429afcb7484518:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T19:14:38,564 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:14:38,564 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T19:14:38,564 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d23b79ada6d885c09c429afcb7484518:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T19:14:38,564 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:14:38,564 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d23b79ada6d885c09c429afcb7484518:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T19:14:38,564 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T19:14:38,564 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T19:14:38,565 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103277 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T19:14:38,565 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T19:14:38,565 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1541): d23b79ada6d885c09c429afcb7484518/A is initiating minor compaction (all files) 2024-11-26T19:14:38,565 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1541): d23b79ada6d885c09c429afcb7484518/B is initiating minor compaction (all files) 2024-11-26T19:14:38,565 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d23b79ada6d885c09c429afcb7484518/B in TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. 2024-11-26T19:14:38,565 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d23b79ada6d885c09c429afcb7484518/A in TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. 2024-11-26T19:14:38,566 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/290007cee1454f9fa36fa5a9062f90d9, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/1016ccddb6d94ca4aaa8f83292074fd3, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/5173fca9e04d469e9814c5043a47b8ed] into tmpdir=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp, totalSize=36.8 K 2024-11-26T19:14:38,566 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/dfc123e1f18a49ef978a840f82eac11f, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/28618b3b3c864dcb802465d4acebf520, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/4e49b55a13c5477193e5572460b62115] into tmpdir=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp, totalSize=100.9 K 2024-11-26T19:14:38,566 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. 2024-11-26T19:14:38,566 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. files: [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/dfc123e1f18a49ef978a840f82eac11f, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/28618b3b3c864dcb802465d4acebf520, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/4e49b55a13c5477193e5572460b62115] 2024-11-26T19:14:38,566 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting 290007cee1454f9fa36fa5a9062f90d9, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1732648473942 2024-11-26T19:14:38,567 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting dfc123e1f18a49ef978a840f82eac11f, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1732648473942 2024-11-26T19:14:38,567 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting 1016ccddb6d94ca4aaa8f83292074fd3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=341, earliestPutTs=1732648474724 2024-11-26T19:14:38,567 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting 28618b3b3c864dcb802465d4acebf520, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=341, earliestPutTs=1732648474724 2024-11-26T19:14:38,568 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4e49b55a13c5477193e5572460b62115, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=361, earliestPutTs=1732648475431 2024-11-26T19:14:38,568 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting 5173fca9e04d469e9814c5043a47b8ed, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=361, earliestPutTs=1732648475439 2024-11-26T19:14:38,579 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d23b79ada6d885c09c429afcb7484518#B#compaction#197 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T19:14:38,580 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/B/ec2130b9fd5c4c4f9ebf8223d8492a33 is 50, key is test_row_0/B:col10/1732648477606/Put/seqid=0 2024-11-26T19:14:38,582 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d23b79ada6d885c09c429afcb7484518] 2024-11-26T19:14:38,610 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241126940f19d3537948298c778829e2e93c1e_d23b79ada6d885c09c429afcb7484518 store=[table=TestAcidGuarantees family=A region=d23b79ada6d885c09c429afcb7484518] 2024-11-26T19:14:38,612 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241126940f19d3537948298c778829e2e93c1e_d23b79ada6d885c09c429afcb7484518, store=[table=TestAcidGuarantees family=A region=d23b79ada6d885c09c429afcb7484518] 2024-11-26T19:14:38,612 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126940f19d3537948298c778829e2e93c1e_d23b79ada6d885c09c429afcb7484518 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d23b79ada6d885c09c429afcb7484518] 2024-11-26T19:14:38,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742065_1241 (size=4469) 2024-11-26T19:14:38,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742064_1240 (size=13221) 2024-11-26T19:14:38,626 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d23b79ada6d885c09c429afcb7484518#A#compaction#198 average throughput is 0.56 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T19:14:38,627 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/A/e12960d9533447059647b1159496b0d9 is 175, key is test_row_0/A:col10/1732648477606/Put/seqid=0 2024-11-26T19:14:38,631 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/B/ec2130b9fd5c4c4f9ebf8223d8492a33 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/ec2130b9fd5c4c4f9ebf8223d8492a33 2024-11-26T19:14:38,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742066_1242 (size=32175) 2024-11-26T19:14:38,640 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d23b79ada6d885c09c429afcb7484518/B of d23b79ada6d885c09c429afcb7484518 into ec2130b9fd5c4c4f9ebf8223d8492a33(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T19:14:38,640 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d23b79ada6d885c09c429afcb7484518: 2024-11-26T19:14:38,640 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., storeName=d23b79ada6d885c09c429afcb7484518/B, priority=13, startTime=1732648478564; duration=0sec 2024-11-26T19:14:38,640 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T19:14:38,640 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d23b79ada6d885c09c429afcb7484518:B 2024-11-26T19:14:38,640 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T19:14:38,641 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T19:14:38,641 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1541): d23b79ada6d885c09c429afcb7484518/C is initiating minor compaction (all files) 2024-11-26T19:14:38,642 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d23b79ada6d885c09c429afcb7484518/C in TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. 2024-11-26T19:14:38,642 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/9ac2ea240ffd488c9952c3579be4e097, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/fcc5e451cdc04b12afd59db7f94992b2, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/60787e1bebea4a948f6a993bbb262299] into tmpdir=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp, totalSize=36.8 K 2024-11-26T19:14:38,642 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting 9ac2ea240ffd488c9952c3579be4e097, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1732648473942 2024-11-26T19:14:38,642 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting fcc5e451cdc04b12afd59db7f94992b2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=341, earliestPutTs=1732648474724 2024-11-26T19:14:38,644 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting 60787e1bebea4a948f6a993bbb262299, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=361, earliestPutTs=1732648475439 2024-11-26T19:14:38,651 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/A/e12960d9533447059647b1159496b0d9 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/e12960d9533447059647b1159496b0d9 2024-11-26T19:14:38,666 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d23b79ada6d885c09c429afcb7484518/A of d23b79ada6d885c09c429afcb7484518 into e12960d9533447059647b1159496b0d9(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T19:14:38,666 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d23b79ada6d885c09c429afcb7484518: 2024-11-26T19:14:38,666 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., storeName=d23b79ada6d885c09c429afcb7484518/A, priority=13, startTime=1732648478564; duration=0sec 2024-11-26T19:14:38,667 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:14:38,667 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d23b79ada6d885c09c429afcb7484518:A 2024-11-26T19:14:38,673 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d23b79ada6d885c09c429afcb7484518#C#compaction#199 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T19:14:38,674 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/C/a656104fc3bb47bfa0cc3a1b0a687a25 is 50, key is test_row_0/C:col10/1732648477606/Put/seqid=0 2024-11-26T19:14:38,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742067_1243 (size=13221) 2024-11-26T19:14:38,712 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/C/a656104fc3bb47bfa0cc3a1b0a687a25 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/a656104fc3bb47bfa0cc3a1b0a687a25 2024-11-26T19:14:38,718 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d23b79ada6d885c09c429afcb7484518/C of d23b79ada6d885c09c429afcb7484518 into a656104fc3bb47bfa0cc3a1b0a687a25(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T19:14:38,718 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d23b79ada6d885c09c429afcb7484518: 2024-11-26T19:14:38,719 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., storeName=d23b79ada6d885c09c429afcb7484518/C, priority=13, startTime=1732648478564; duration=0sec 2024-11-26T19:14:38,720 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:14:38,720 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d23b79ada6d885c09c429afcb7484518:C 2024-11-26T19:14:38,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-26T19:14:38,752 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T19:14:38,753 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T19:14:38,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-11-26T19:14:38,756 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T19:14:38,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-26T19:14:38,757 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T19:14:38,757 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T19:14:38,785 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d23b79ada6d885c09c429afcb7484518 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-26T19:14:38,786 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d23b79ada6d885c09c429afcb7484518, store=A 2024-11-26T19:14:38,786 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:14:38,786 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d23b79ada6d885c09c429afcb7484518, store=B 2024-11-26T19:14:38,786 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:14:38,786 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d23b79ada6d885c09c429afcb7484518, store=C 2024-11-26T19:14:38,786 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:14:38,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(8855): Flush requested on d23b79ada6d885c09c429afcb7484518 2024-11-26T19:14:38,805 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112671f0df6a11394169835eceb14e713aef_d23b79ada6d885c09c429afcb7484518 is 50, key is test_row_0/A:col10/1732648477642/Put/seqid=0 2024-11-26T19:14:38,822 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:38,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39306 deadline: 1732648538817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:38,823 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:38,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39292 deadline: 1732648538817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:38,823 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:38,823 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:38,823 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:38,824 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:38,824 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:38,824 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:38,824 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:38,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39332 deadline: 1732648538820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:38,824 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:38,824 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:38,824 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:38,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742068_1244 (size=14994) 2024-11-26T19:14:38,835 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:38,840 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112671f0df6a11394169835eceb14e713aef_d23b79ada6d885c09c429afcb7484518 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112671f0df6a11394169835eceb14e713aef_d23b79ada6d885c09c429afcb7484518 2024-11-26T19:14:38,841 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/A/b437db3e2d7744c58757ce0b897864b0, store: [table=TestAcidGuarantees family=A region=d23b79ada6d885c09c429afcb7484518] 2024-11-26T19:14:38,842 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/A/b437db3e2d7744c58757ce0b897864b0 is 175, key is test_row_0/A:col10/1732648477642/Put/seqid=0 2024-11-26T19:14:38,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742069_1245 (size=39949) 2024-11-26T19:14:38,859 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=381, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/A/b437db3e2d7744c58757ce0b897864b0 2024-11-26T19:14:38,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-26T19:14:38,882 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/B/01a7489108b6460fac0e21dbd14e4b5b is 50, key is test_row_0/B:col10/1732648477642/Put/seqid=0 2024-11-26T19:14:38,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742070_1246 (size=12301) 2024-11-26T19:14:38,899 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=381 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/B/01a7489108b6460fac0e21dbd14e4b5b 2024-11-26T19:14:38,907 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/C/a5cfa0e2a0db42fe924f3ff1a21add2c is 50, key is test_row_0/C:col10/1732648477642/Put/seqid=0 2024-11-26T19:14:38,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742071_1247 (size=12301) 2024-11-26T19:14:38,912 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-26T19:14:38,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. 2024-11-26T19:14:38,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. as already flushing 2024-11-26T19:14:38,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. 2024-11-26T19:14:38,913 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => d23b79ada6d885c09c429afcb7484518, NAME => 'TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:38,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => d23b79ada6d885c09c429afcb7484518, NAME => 'TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:38,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d23b79ada6d885c09c429afcb7484518, NAME => 'TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d23b79ada6d885c09c429afcb7484518, NAME => 'TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:38,936 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:38,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39306 deadline: 1732648538932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:38,937 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:38,937 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:38,937 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:38,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39292 deadline: 1732648538932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:38,937 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:38,937 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:38,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39332 deadline: 1732648538932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:38,937 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:38,937 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:38,938 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:38,938 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:38,938 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:38,938 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:39,066 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-26T19:14:39,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. 2024-11-26T19:14:39,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. as already flushing 2024-11-26T19:14:39,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. 2024-11-26T19:14:39,066 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => d23b79ada6d885c09c429afcb7484518, NAME => 'TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:39,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => d23b79ada6d885c09c429afcb7484518, NAME => 'TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:39,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d23b79ada6d885c09c429afcb7484518, NAME => 'TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d23b79ada6d885c09c429afcb7484518, NAME => 'TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:39,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-26T19:14:39,143 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:39,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39306 deadline: 1732648539142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:39,144 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:39,144 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:39,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39292 deadline: 1732648539142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:39,144 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:39,144 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:39,144 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:39,145 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:39,145 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:39,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39332 deadline: 1732648539142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:39,145 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:39,145 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:39,145 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:39,145 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:39,219 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-26T19:14:39,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. 2024-11-26T19:14:39,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. as already flushing 2024-11-26T19:14:39,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. 2024-11-26T19:14:39,219 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => d23b79ada6d885c09c429afcb7484518, NAME => 'TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:39,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => d23b79ada6d885c09c429afcb7484518, NAME => 'TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:39,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d23b79ada6d885c09c429afcb7484518, NAME => 'TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d23b79ada6d885c09c429afcb7484518, NAME => 'TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:39,312 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=381 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/C/a5cfa0e2a0db42fe924f3ff1a21add2c 2024-11-26T19:14:39,317 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/A/b437db3e2d7744c58757ce0b897864b0 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/b437db3e2d7744c58757ce0b897864b0 2024-11-26T19:14:39,323 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/b437db3e2d7744c58757ce0b897864b0, entries=200, sequenceid=381, filesize=39.0 K 2024-11-26T19:14:39,324 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/B/01a7489108b6460fac0e21dbd14e4b5b as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/01a7489108b6460fac0e21dbd14e4b5b 2024-11-26T19:14:39,333 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/01a7489108b6460fac0e21dbd14e4b5b, entries=150, sequenceid=381, filesize=12.0 K 2024-11-26T19:14:39,339 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/C/a5cfa0e2a0db42fe924f3ff1a21add2c as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/a5cfa0e2a0db42fe924f3ff1a21add2c 2024-11-26T19:14:39,354 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/a5cfa0e2a0db42fe924f3ff1a21add2c, entries=150, sequenceid=381, filesize=12.0 K 2024-11-26T19:14:39,357 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for d23b79ada6d885c09c429afcb7484518 in 571ms, sequenceid=381, compaction requested=false 2024-11-26T19:14:39,357 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d23b79ada6d885c09c429afcb7484518: 2024-11-26T19:14:39,372 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-26T19:14:39,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. 2024-11-26T19:14:39,373 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing d23b79ada6d885c09c429afcb7484518 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-26T19:14:39,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d23b79ada6d885c09c429afcb7484518, store=A 2024-11-26T19:14:39,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:14:39,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d23b79ada6d885c09c429afcb7484518, store=B 2024-11-26T19:14:39,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:14:39,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d23b79ada6d885c09c429afcb7484518, store=C 2024-11-26T19:14:39,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:14:39,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-26T19:14:39,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126b60e6ff4e1794913840a3fc9346f6aaf_d23b79ada6d885c09c429afcb7484518 is 50, key is test_row_0/A:col10/1732648478812/Put/seqid=0 2024-11-26T19:14:39,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742072_1248 (size=12454) 2024-11-26T19:14:39,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(8855): Flush requested on d23b79ada6d885c09c429afcb7484518 2024-11-26T19:14:39,454 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. as already flushing 2024-11-26T19:14:39,487 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:39,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39332 deadline: 1732648539478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:39,488 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:39,488 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:39,488 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:39,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39306 deadline: 1732648539480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:39,488 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:39,488 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:39,488 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:39,488 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:39,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39292 deadline: 1732648539487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:39,488 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:39,489 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:39,489 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:39,489 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:39,593 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:39,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39332 deadline: 1732648539592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:39,593 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:39,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39306 deadline: 1732648539592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:39,594 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:39,594 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:39,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39292 deadline: 1732648539592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:39,594 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:39,594 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:39,594 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:39,594 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:39,594 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:39,594 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:39,594 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:39,594 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:39,641 INFO [Thread-665 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-665. 2024-11-26T19:14:39,641 DEBUG [Thread-665 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T19:14:39,641 DEBUG [Thread-665 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:14:39,641 DEBUG [Thread-665 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:14:39,641 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T19:14:39,647 INFO [Thread-663 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-663. 2024-11-26T19:14:39,648 DEBUG [Thread-663 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T19:14:39,648 DEBUG [Thread-663 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:14:39,648 DEBUG [Thread-663 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:14:39,648 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T19:14:39,657 INFO [Thread-669 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-669. 2024-11-26T19:14:39,657 INFO [Thread-667 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-667. 2024-11-26T19:14:39,657 DEBUG [Thread-667 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T19:14:39,657 DEBUG [Thread-669 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T19:14:39,657 DEBUG [Thread-667 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:14:39,657 DEBUG [Thread-669 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:14:39,657 DEBUG [Thread-667 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:14:39,657 DEBUG [Thread-669 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:14:39,657 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T19:14:39,657 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T19:14:39,802 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:39,802 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:39,802 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:39,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39306 deadline: 1732648539802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:39,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39332 deadline: 1732648539802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:39,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39292 deadline: 1732648539802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:39,803 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:39,803 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:39,803 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:39,803 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:39,803 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:39,803 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:39,805 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:39,805 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:39,805 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:39,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:39,818 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126b60e6ff4e1794913840a3fc9346f6aaf_d23b79ada6d885c09c429afcb7484518 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126b60e6ff4e1794913840a3fc9346f6aaf_d23b79ada6d885c09c429afcb7484518 2024-11-26T19:14:39,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/A/8c028ce2f77d42a09b395964a2e95be0, store: [table=TestAcidGuarantees family=A region=d23b79ada6d885c09c429afcb7484518] 2024-11-26T19:14:39,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/A/8c028ce2f77d42a09b395964a2e95be0 is 175, key is test_row_0/A:col10/1732648478812/Put/seqid=0 2024-11-26T19:14:39,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742073_1249 (size=31255) 2024-11-26T19:14:39,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-26T19:14:40,113 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:40,113 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:40,113 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:40,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39292 deadline: 1732648540112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:40,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39332 deadline: 1732648540113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:40,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39306 deadline: 1732648540112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:40,113 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:40,113 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:40,113 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:40,113 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:40,113 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:40,114 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:40,114 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:40,114 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:40,114 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:40,240 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=396, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/A/8c028ce2f77d42a09b395964a2e95be0 2024-11-26T19:14:40,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/B/64d77f8bc3174571ad7dadfef44c591f is 50, key is test_row_0/B:col10/1732648478812/Put/seqid=0 2024-11-26T19:14:40,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742074_1250 (size=12301) 2024-11-26T19:14:40,622 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:40,622 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:40,622 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:40,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39332 deadline: 1732648540622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:40,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39306 deadline: 1732648540622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:40,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39292 deadline: 1732648540622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:40,623 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:40,623 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:40,623 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:40,623 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:40,623 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:40,623 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:40,623 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:40,623 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:40,623 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:14:40,659 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=396 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/B/64d77f8bc3174571ad7dadfef44c591f 2024-11-26T19:14:40,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/C/bdc9a277f358493185cd752eeec97c4e is 50, key is test_row_0/C:col10/1732648478812/Put/seqid=0 2024-11-26T19:14:40,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742075_1251 (size=12301) 2024-11-26T19:14:40,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-26T19:14:41,072 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=396 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/C/bdc9a277f358493185cd752eeec97c4e 2024-11-26T19:14:41,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/A/8c028ce2f77d42a09b395964a2e95be0 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/8c028ce2f77d42a09b395964a2e95be0 2024-11-26T19:14:41,080 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/8c028ce2f77d42a09b395964a2e95be0, entries=150, sequenceid=396, filesize=30.5 K 2024-11-26T19:14:41,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/B/64d77f8bc3174571ad7dadfef44c591f as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/64d77f8bc3174571ad7dadfef44c591f 2024-11-26T19:14:41,085 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/64d77f8bc3174571ad7dadfef44c591f, entries=150, sequenceid=396, filesize=12.0 K 2024-11-26T19:14:41,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/C/bdc9a277f358493185cd752eeec97c4e as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/bdc9a277f358493185cd752eeec97c4e 2024-11-26T19:14:41,090 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/bdc9a277f358493185cd752eeec97c4e, entries=150, sequenceid=396, filesize=12.0 K 2024-11-26T19:14:41,091 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for d23b79ada6d885c09c429afcb7484518 in 1718ms, sequenceid=396, compaction requested=true 2024-11-26T19:14:41,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for d23b79ada6d885c09c429afcb7484518: 2024-11-26T19:14:41,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. 2024-11-26T19:14:41,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-11-26T19:14:41,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-11-26T19:14:41,094 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-26T19:14:41,094 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3360 sec 2024-11-26T19:14:41,096 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 2.3420 sec 2024-11-26T19:14:41,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(8855): Flush requested on d23b79ada6d885c09c429afcb7484518 2024-11-26T19:14:41,633 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d23b79ada6d885c09c429afcb7484518 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-26T19:14:41,633 INFO [Thread-652 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-652. 2024-11-26T19:14:41,633 DEBUG [Thread-652 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T19:14:41,633 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d23b79ada6d885c09c429afcb7484518, store=A 2024-11-26T19:14:41,633 DEBUG [Thread-652 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:14:41,633 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:14:41,634 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d23b79ada6d885c09c429afcb7484518, store=B 2024-11-26T19:14:41,634 DEBUG [Thread-652 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:14:41,634 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:14:41,634 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T19:14:41,634 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d23b79ada6d885c09c429afcb7484518, store=C 2024-11-26T19:14:41,634 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:14:41,640 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126f50be5b5415343a686434b851ff20c66_d23b79ada6d885c09c429afcb7484518 is 50, key is test_row_0/A:col10/1732648481632/Put/seqid=0 2024-11-26T19:14:41,644 INFO [Thread-658 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-658. 2024-11-26T19:14:41,644 DEBUG [Thread-658 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T19:14:41,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742076_1252 (size=12454) 2024-11-26T19:14:41,644 DEBUG [Thread-658 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:14:41,644 DEBUG [Thread-658 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:14:41,644 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T19:14:41,644 INFO [Thread-654 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-654. 2024-11-26T19:14:41,644 DEBUG [Thread-654 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T19:14:41,644 DEBUG [Thread-654 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:14:41,644 DEBUG [Thread-654 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:14:41,644 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T19:14:42,045 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:42,049 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126f50be5b5415343a686434b851ff20c66_d23b79ada6d885c09c429afcb7484518 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126f50be5b5415343a686434b851ff20c66_d23b79ada6d885c09c429afcb7484518 2024-11-26T19:14:42,050 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/A/fc0a8189af804701893b092b2dbe9395, store: [table=TestAcidGuarantees family=A region=d23b79ada6d885c09c429afcb7484518] 2024-11-26T19:14:42,051 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/A/fc0a8189af804701893b092b2dbe9395 is 175, key is test_row_0/A:col10/1732648481632/Put/seqid=0 2024-11-26T19:14:42,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742077_1253 (size=31255) 2024-11-26T19:14:42,135 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-26T19:14:42,253 INFO [Thread-660 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-660. 2024-11-26T19:14:42,253 DEBUG [Thread-660 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T19:14:42,253 DEBUG [Thread-660 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:14:42,253 DEBUG [Thread-660 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:14:42,253 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T19:14:42,263 INFO [Thread-656 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-656. 2024-11-26T19:14:42,263 DEBUG [Thread-656 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T19:14:42,263 DEBUG [Thread-656 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:14:42,263 DEBUG [Thread-656 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:14:42,263 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T19:14:42,461 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=414, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/A/fc0a8189af804701893b092b2dbe9395 2024-11-26T19:14:42,468 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/B/89bcd3e7a1454002b2b9b001bd940ee5 is 50, key is test_row_0/B:col10/1732648481632/Put/seqid=0 2024-11-26T19:14:42,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742078_1254 (size=12301) 2024-11-26T19:14:42,480 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=414 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/B/89bcd3e7a1454002b2b9b001bd940ee5 2024-11-26T19:14:42,488 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/C/e00cb2156de043b1b5695f2f97c9b8b6 is 50, key is test_row_0/C:col10/1732648481632/Put/seqid=0 2024-11-26T19:14:42,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742079_1255 (size=12301) 2024-11-26T19:14:42,892 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=414 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/C/e00cb2156de043b1b5695f2f97c9b8b6 2024-11-26T19:14:42,898 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/A/fc0a8189af804701893b092b2dbe9395 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/fc0a8189af804701893b092b2dbe9395 2024-11-26T19:14:42,903 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/fc0a8189af804701893b092b2dbe9395, entries=150, sequenceid=414, filesize=30.5 K 2024-11-26T19:14:42,904 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/B/89bcd3e7a1454002b2b9b001bd940ee5 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/89bcd3e7a1454002b2b9b001bd940ee5 2024-11-26T19:14:42,907 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/89bcd3e7a1454002b2b9b001bd940ee5, entries=150, sequenceid=414, filesize=12.0 K 2024-11-26T19:14:42,908 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/C/e00cb2156de043b1b5695f2f97c9b8b6 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/e00cb2156de043b1b5695f2f97c9b8b6 2024-11-26T19:14:42,911 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/e00cb2156de043b1b5695f2f97c9b8b6, entries=150, sequenceid=414, filesize=12.0 K 2024-11-26T19:14:42,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-26T19:14:42,912 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T19:14:42,912 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=26.84 KB/27480 for d23b79ada6d885c09c429afcb7484518 in 1279ms, sequenceid=414, compaction requested=true 2024-11-26T19:14:42,912 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d23b79ada6d885c09c429afcb7484518: 2024-11-26T19:14:42,912 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-26T19:14:42,912 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 114 2024-11-26T19:14:42,912 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 72 2024-11-26T19:14:42,912 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 33 2024-11-26T19:14:42,912 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d23b79ada6d885c09c429afcb7484518:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T19:14:42,912 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 70 2024-11-26T19:14:42,912 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 32 2024-11-26T19:14:42,912 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-26T19:14:42,912 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:14:42,912 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2617 2024-11-26T19:14:42,913 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2533 2024-11-26T19:14:42,913 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T19:14:42,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d23b79ada6d885c09c429afcb7484518:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T19:14:42,913 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-26T19:14:42,913 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1142 2024-11-26T19:14:42,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:14:42,913 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3425 rows 2024-11-26T19:14:42,913 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T19:14:42,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d23b79ada6d885c09c429afcb7484518:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T19:14:42,913 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1132 2024-11-26T19:14:42,913 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3391 rows 2024-11-26T19:14:42,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T19:14:42,913 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-26T19:14:42,913 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-26T19:14:42,913 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:14:42,913 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:14:42,913 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T19:14:42,914 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-26T19:14:42,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-26T19:14:42,919 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50124 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T19:14:42,919 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 134634 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T19:14:42,919 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1541): d23b79ada6d885c09c429afcb7484518/A is initiating minor compaction (all files) 2024-11-26T19:14:42,919 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1541): d23b79ada6d885c09c429afcb7484518/B is initiating minor compaction (all files) 2024-11-26T19:14:42,919 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d23b79ada6d885c09c429afcb7484518/A in TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. 2024-11-26T19:14:42,919 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d23b79ada6d885c09c429afcb7484518/B in TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. 2024-11-26T19:14:42,919 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/e12960d9533447059647b1159496b0d9, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/b437db3e2d7744c58757ce0b897864b0, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/8c028ce2f77d42a09b395964a2e95be0, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/fc0a8189af804701893b092b2dbe9395] into tmpdir=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp, totalSize=131.5 K 2024-11-26T19:14:42,919 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/ec2130b9fd5c4c4f9ebf8223d8492a33, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/01a7489108b6460fac0e21dbd14e4b5b, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/64d77f8bc3174571ad7dadfef44c591f, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/89bcd3e7a1454002b2b9b001bd940ee5] into tmpdir=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp, totalSize=48.9 K 2024-11-26T19:14:42,919 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. 2024-11-26T19:14:42,920 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. files: [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/e12960d9533447059647b1159496b0d9, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/b437db3e2d7744c58757ce0b897864b0, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/8c028ce2f77d42a09b395964a2e95be0, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/fc0a8189af804701893b092b2dbe9395] 2024-11-26T19:14:42,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-26T19:14:42,920 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting ec2130b9fd5c4c4f9ebf8223d8492a33, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=361, earliestPutTs=1732648475439 2024-11-26T19:14:42,920 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting e12960d9533447059647b1159496b0d9, keycount=150, bloomtype=ROW, size=31.4 K, encoding=NONE, compression=NONE, seqNum=361, earliestPutTs=1732648475439 2024-11-26T19:14:42,921 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting 01a7489108b6460fac0e21dbd14e4b5b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=381, earliestPutTs=1732648477642 2024-11-26T19:14:42,921 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting b437db3e2d7744c58757ce0b897864b0, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=381, earliestPutTs=1732648477625 2024-11-26T19:14:42,921 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732648482921"}]},"ts":"1732648482921"} 2024-11-26T19:14:42,921 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8c028ce2f77d42a09b395964a2e95be0, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=396, earliestPutTs=1732648478812 2024-11-26T19:14:42,921 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting 64d77f8bc3174571ad7dadfef44c591f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=396, earliestPutTs=1732648478812 2024-11-26T19:14:42,922 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting fc0a8189af804701893b092b2dbe9395, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=414, earliestPutTs=1732648479474 2024-11-26T19:14:42,922 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting 89bcd3e7a1454002b2b9b001bd940ee5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=414, earliestPutTs=1732648479474 2024-11-26T19:14:42,923 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-26T19:14:42,923 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-26T19:14:42,924 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-26T19:14:42,926 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=54, ppid=53, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d23b79ada6d885c09c429afcb7484518, UNASSIGN}] 2024-11-26T19:14:42,927 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=54, ppid=53, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d23b79ada6d885c09c429afcb7484518, UNASSIGN 2024-11-26T19:14:42,928 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=54 updating hbase:meta row=d23b79ada6d885c09c429afcb7484518, regionState=CLOSING, regionLocation=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:42,929 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=54, ppid=53, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d23b79ada6d885c09c429afcb7484518, UNASSIGN because future has completed 2024-11-26T19:14:42,930 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-26T19:14:42,930 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; CloseRegionProcedure d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970}] 2024-11-26T19:14:42,941 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d23b79ada6d885c09c429afcb7484518] 2024-11-26T19:14:42,943 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d23b79ada6d885c09c429afcb7484518#B#compaction#209 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T19:14:42,944 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241126bc5ce9d7b2814d799041ebb201df14c9_d23b79ada6d885c09c429afcb7484518 store=[table=TestAcidGuarantees family=A region=d23b79ada6d885c09c429afcb7484518] 2024-11-26T19:14:42,944 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/B/b450554817554703bf4eaf50688ed8f7 is 50, key is test_row_0/B:col10/1732648481632/Put/seqid=0 2024-11-26T19:14:42,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742080_1256 (size=13357) 2024-11-26T19:14:42,963 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/B/b450554817554703bf4eaf50688ed8f7 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/b450554817554703bf4eaf50688ed8f7 2024-11-26T19:14:42,969 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d23b79ada6d885c09c429afcb7484518/B of d23b79ada6d885c09c429afcb7484518 into b450554817554703bf4eaf50688ed8f7(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T19:14:42,969 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d23b79ada6d885c09c429afcb7484518: 2024-11-26T19:14:42,969 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., storeName=d23b79ada6d885c09c429afcb7484518/B, priority=12, startTime=1732648482912; duration=0sec 2024-11-26T19:14:42,969 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T19:14:42,969 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d23b79ada6d885c09c429afcb7484518:B 2024-11-26T19:14:42,969 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T19:14:42,970 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50124 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T19:14:42,970 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1541): d23b79ada6d885c09c429afcb7484518/C is initiating minor compaction (all files) 2024-11-26T19:14:42,971 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d23b79ada6d885c09c429afcb7484518/C in TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. 2024-11-26T19:14:42,971 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/a656104fc3bb47bfa0cc3a1b0a687a25, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/a5cfa0e2a0db42fe924f3ff1a21add2c, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/bdc9a277f358493185cd752eeec97c4e, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/e00cb2156de043b1b5695f2f97c9b8b6] into tmpdir=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp, totalSize=48.9 K 2024-11-26T19:14:42,971 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting a656104fc3bb47bfa0cc3a1b0a687a25, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=361, earliestPutTs=1732648475439 2024-11-26T19:14:42,972 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting a5cfa0e2a0db42fe924f3ff1a21add2c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=381, earliestPutTs=1732648477642 2024-11-26T19:14:42,972 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting bdc9a277f358493185cd752eeec97c4e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=396, earliestPutTs=1732648478812 2024-11-26T19:14:42,972 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting e00cb2156de043b1b5695f2f97c9b8b6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=414, earliestPutTs=1732648479474 2024-11-26T19:14:42,984 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241126bc5ce9d7b2814d799041ebb201df14c9_d23b79ada6d885c09c429afcb7484518, store=[table=TestAcidGuarantees family=A region=d23b79ada6d885c09c429afcb7484518] 2024-11-26T19:14:42,985 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126bc5ce9d7b2814d799041ebb201df14c9_d23b79ada6d885c09c429afcb7484518 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d23b79ada6d885c09c429afcb7484518] 2024-11-26T19:14:42,991 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d23b79ada6d885c09c429afcb7484518#C#compaction#211 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T19:14:42,992 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/C/81f58b99d3144cdc9b501bc4583e4bf6 is 50, key is test_row_0/C:col10/1732648481632/Put/seqid=0 2024-11-26T19:14:43,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742081_1257 (size=4469) 2024-11-26T19:14:43,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-26T19:14:43,023 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d23b79ada6d885c09c429afcb7484518#A#compaction#210 average throughput is 0.30 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T19:14:43,023 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/A/27b103e1563e4e5585059fd805f66860 is 175, key is test_row_0/A:col10/1732648481632/Put/seqid=0 2024-11-26T19:14:43,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742082_1258 (size=13357) 2024-11-26T19:14:43,040 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/C/81f58b99d3144cdc9b501bc4583e4bf6 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/81f58b99d3144cdc9b501bc4583e4bf6 2024-11-26T19:14:43,046 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d23b79ada6d885c09c429afcb7484518/C of d23b79ada6d885c09c429afcb7484518 into 81f58b99d3144cdc9b501bc4583e4bf6(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T19:14:43,046 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d23b79ada6d885c09c429afcb7484518: 2024-11-26T19:14:43,046 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., storeName=d23b79ada6d885c09c429afcb7484518/C, priority=12, startTime=1732648482913; duration=0sec 2024-11-26T19:14:43,046 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:14:43,046 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d23b79ada6d885c09c429afcb7484518:C 2024-11-26T19:14:43,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742083_1259 (size=32311) 2024-11-26T19:14:43,083 INFO [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] handler.UnassignRegionHandler(122): Close d23b79ada6d885c09c429afcb7484518 2024-11-26T19:14:43,083 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-26T19:14:43,083 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1722): Closing d23b79ada6d885c09c429afcb7484518, disabling compactions & flushes 2024-11-26T19:14:43,083 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1993): waiting for 1 compactions to complete for region TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. 2024-11-26T19:14:43,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-26T19:14:43,470 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/A/27b103e1563e4e5585059fd805f66860 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/27b103e1563e4e5585059fd805f66860 2024-11-26T19:14:43,475 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d23b79ada6d885c09c429afcb7484518/A of d23b79ada6d885c09c429afcb7484518 into 27b103e1563e4e5585059fd805f66860(size=31.6 K), total size for store is 31.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T19:14:43,475 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d23b79ada6d885c09c429afcb7484518: 2024-11-26T19:14:43,475 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., storeName=d23b79ada6d885c09c429afcb7484518/A, priority=12, startTime=1732648482912; duration=0sec 2024-11-26T19:14:43,475 INFO [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. 2024-11-26T19:14:43,475 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:14:43,475 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. 2024-11-26T19:14:43,475 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d23b79ada6d885c09c429afcb7484518:A 2024-11-26T19:14:43,475 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. after waiting 0 ms 2024-11-26T19:14:43,475 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. 2024-11-26T19:14:43,475 INFO [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(2902): Flushing d23b79ada6d885c09c429afcb7484518 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-26T19:14:43,476 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d23b79ada6d885c09c429afcb7484518, store=A 2024-11-26T19:14:43,476 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:14:43,476 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d23b79ada6d885c09c429afcb7484518, store=B 2024-11-26T19:14:43,476 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:14:43,476 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d23b79ada6d885c09c429afcb7484518, store=C 2024-11-26T19:14:43,476 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:14:43,489 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126931396bb95f8476eb7f2f7aef7d53692_d23b79ada6d885c09c429afcb7484518 is 50, key is test_row_0/A:col10/1732648482262/Put/seqid=0 2024-11-26T19:14:43,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742084_1260 (size=12454) 2024-11-26T19:14:43,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-26T19:14:43,902 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:43,906 INFO [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126931396bb95f8476eb7f2f7aef7d53692_d23b79ada6d885c09c429afcb7484518 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126931396bb95f8476eb7f2f7aef7d53692_d23b79ada6d885c09c429afcb7484518 2024-11-26T19:14:43,907 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/A/824e302f866b45a9b6edbed9ce760038, store: [table=TestAcidGuarantees family=A region=d23b79ada6d885c09c429afcb7484518] 2024-11-26T19:14:43,908 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/A/824e302f866b45a9b6edbed9ce760038 is 175, key is test_row_0/A:col10/1732648482262/Put/seqid=0 2024-11-26T19:14:43,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742085_1261 (size=31255) 2024-11-26T19:14:44,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-26T19:14:44,312 INFO [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=424, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/A/824e302f866b45a9b6edbed9ce760038 2024-11-26T19:14:44,320 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/B/f7dc3bd6baca4d4681e8631e4249df42 is 50, key is test_row_0/B:col10/1732648482262/Put/seqid=0 2024-11-26T19:14:44,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742086_1262 (size=12301) 2024-11-26T19:14:44,729 INFO [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=424 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/B/f7dc3bd6baca4d4681e8631e4249df42 2024-11-26T19:14:44,736 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/C/e310ede3e8414ebcabab58397d8984eb is 50, key is test_row_0/C:col10/1732648482262/Put/seqid=0 2024-11-26T19:14:44,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742087_1263 (size=12301) 2024-11-26T19:14:45,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-26T19:14:45,140 INFO [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=424 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/C/e310ede3e8414ebcabab58397d8984eb 2024-11-26T19:14:45,145 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/A/824e302f866b45a9b6edbed9ce760038 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/824e302f866b45a9b6edbed9ce760038 2024-11-26T19:14:45,149 INFO [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/824e302f866b45a9b6edbed9ce760038, entries=150, sequenceid=424, filesize=30.5 K 2024-11-26T19:14:45,150 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/B/f7dc3bd6baca4d4681e8631e4249df42 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/f7dc3bd6baca4d4681e8631e4249df42 2024-11-26T19:14:45,153 INFO [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/f7dc3bd6baca4d4681e8631e4249df42, entries=150, sequenceid=424, filesize=12.0 K 2024-11-26T19:14:45,154 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/.tmp/C/e310ede3e8414ebcabab58397d8984eb as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/e310ede3e8414ebcabab58397d8984eb 2024-11-26T19:14:45,158 INFO [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/e310ede3e8414ebcabab58397d8984eb, entries=150, sequenceid=424, filesize=12.0 K 2024-11-26T19:14:45,159 INFO [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for d23b79ada6d885c09c429afcb7484518 in 1684ms, sequenceid=424, compaction requested=false 2024-11-26T19:14:45,159 DEBUG [StoreCloser-TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/da470e667c0d413c9473b3bd29f894ac, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/d96b98aca1e442a1b92272f6a117f7e5, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/ca4f75ea08b041fcabc7c158fe66b6b6, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/fba6c4a30c5945ae940f8322cad0734a, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/2be2eaf6a9a749f38651fb10444f3051, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/f0250d58167649bcb36eb05b988177ee, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/8674593c15d3428e8efbbc7cbaa9cea9, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/e95b58a058f248d6bd88a161f440e1fc, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/0f6d6c4fad404213a06e831a323e3c45, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/01abae871b874ce8810a2a437c939142, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/c33f837a479b44d99752ae594c8c2e20, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/2868ae3397044c76bea4e911b36b2a6d, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/4c8b5e337b094684834986dd87123dbe, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/3dde213ba3e84412810eaef5d9bb1299, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/e04f1b3aa4e343f387ae7eec9cf362b5, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/96a41b21e7a942d793f3f66eabe8d937, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/a0b8c1e8d50e41039eb047ec9f582621, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/bbc93e0962c14986ad2d6f3a781eea95, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/8f1d34e65e1947c49f150925f1a3cd26, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/fbafe2f643924acca20660e5b3ceac02, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/e8311f3d5af74e5e81597a7261fb4780, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/7e33ffdee4e7427e86868e6b19b30ff4, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/8d976e9543b347658e3260e323c8fba0, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/048a166dcbec4c2c9cf2bdbea89fed41, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/dfc123e1f18a49ef978a840f82eac11f, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/28618b3b3c864dcb802465d4acebf520, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/4e49b55a13c5477193e5572460b62115, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/e12960d9533447059647b1159496b0d9, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/b437db3e2d7744c58757ce0b897864b0, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/8c028ce2f77d42a09b395964a2e95be0, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/fc0a8189af804701893b092b2dbe9395] to archive 2024-11-26T19:14:45,160 DEBUG [StoreCloser-TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-26T19:14:45,162 DEBUG [StoreCloser-TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/da470e667c0d413c9473b3bd29f894ac to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/da470e667c0d413c9473b3bd29f894ac 2024-11-26T19:14:45,164 DEBUG [StoreCloser-TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/d96b98aca1e442a1b92272f6a117f7e5 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/d96b98aca1e442a1b92272f6a117f7e5 2024-11-26T19:14:45,165 DEBUG [StoreCloser-TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/ca4f75ea08b041fcabc7c158fe66b6b6 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/ca4f75ea08b041fcabc7c158fe66b6b6 2024-11-26T19:14:45,166 DEBUG [StoreCloser-TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/fba6c4a30c5945ae940f8322cad0734a to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/fba6c4a30c5945ae940f8322cad0734a 2024-11-26T19:14:45,168 DEBUG [StoreCloser-TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/2be2eaf6a9a749f38651fb10444f3051 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/2be2eaf6a9a749f38651fb10444f3051 2024-11-26T19:14:45,169 DEBUG [StoreCloser-TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/f0250d58167649bcb36eb05b988177ee to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/f0250d58167649bcb36eb05b988177ee 2024-11-26T19:14:45,170 DEBUG [StoreCloser-TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/8674593c15d3428e8efbbc7cbaa9cea9 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/8674593c15d3428e8efbbc7cbaa9cea9 2024-11-26T19:14:45,172 DEBUG [StoreCloser-TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/e95b58a058f248d6bd88a161f440e1fc to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/e95b58a058f248d6bd88a161f440e1fc 2024-11-26T19:14:45,173 DEBUG [StoreCloser-TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/0f6d6c4fad404213a06e831a323e3c45 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/0f6d6c4fad404213a06e831a323e3c45 2024-11-26T19:14:45,174 DEBUG [StoreCloser-TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/01abae871b874ce8810a2a437c939142 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/01abae871b874ce8810a2a437c939142 2024-11-26T19:14:45,176 DEBUG [StoreCloser-TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/c33f837a479b44d99752ae594c8c2e20 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/c33f837a479b44d99752ae594c8c2e20 2024-11-26T19:14:45,177 DEBUG [StoreCloser-TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/2868ae3397044c76bea4e911b36b2a6d to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/2868ae3397044c76bea4e911b36b2a6d 2024-11-26T19:14:45,178 DEBUG [StoreCloser-TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/4c8b5e337b094684834986dd87123dbe to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/4c8b5e337b094684834986dd87123dbe 2024-11-26T19:14:45,179 DEBUG [StoreCloser-TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/3dde213ba3e84412810eaef5d9bb1299 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/3dde213ba3e84412810eaef5d9bb1299 2024-11-26T19:14:45,180 DEBUG [StoreCloser-TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/e04f1b3aa4e343f387ae7eec9cf362b5 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/e04f1b3aa4e343f387ae7eec9cf362b5 2024-11-26T19:14:45,182 DEBUG [StoreCloser-TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/96a41b21e7a942d793f3f66eabe8d937 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/96a41b21e7a942d793f3f66eabe8d937 2024-11-26T19:14:45,183 DEBUG [StoreCloser-TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/a0b8c1e8d50e41039eb047ec9f582621 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/a0b8c1e8d50e41039eb047ec9f582621 2024-11-26T19:14:45,184 DEBUG [StoreCloser-TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/bbc93e0962c14986ad2d6f3a781eea95 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/bbc93e0962c14986ad2d6f3a781eea95 2024-11-26T19:14:45,185 DEBUG [StoreCloser-TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/8f1d34e65e1947c49f150925f1a3cd26 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/8f1d34e65e1947c49f150925f1a3cd26 2024-11-26T19:14:45,186 DEBUG [StoreCloser-TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/fbafe2f643924acca20660e5b3ceac02 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/fbafe2f643924acca20660e5b3ceac02 2024-11-26T19:14:45,187 DEBUG [StoreCloser-TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/e8311f3d5af74e5e81597a7261fb4780 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/e8311f3d5af74e5e81597a7261fb4780 2024-11-26T19:14:45,188 DEBUG [StoreCloser-TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/7e33ffdee4e7427e86868e6b19b30ff4 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/7e33ffdee4e7427e86868e6b19b30ff4 2024-11-26T19:14:45,190 DEBUG [StoreCloser-TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/8d976e9543b347658e3260e323c8fba0 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/8d976e9543b347658e3260e323c8fba0 2024-11-26T19:14:45,191 DEBUG [StoreCloser-TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/048a166dcbec4c2c9cf2bdbea89fed41 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/048a166dcbec4c2c9cf2bdbea89fed41 2024-11-26T19:14:45,192 DEBUG [StoreCloser-TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/dfc123e1f18a49ef978a840f82eac11f to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/dfc123e1f18a49ef978a840f82eac11f 2024-11-26T19:14:45,193 DEBUG [StoreCloser-TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/28618b3b3c864dcb802465d4acebf520 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/28618b3b3c864dcb802465d4acebf520 2024-11-26T19:14:45,194 DEBUG [StoreCloser-TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/4e49b55a13c5477193e5572460b62115 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/4e49b55a13c5477193e5572460b62115 2024-11-26T19:14:45,195 DEBUG [StoreCloser-TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/e12960d9533447059647b1159496b0d9 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/e12960d9533447059647b1159496b0d9 2024-11-26T19:14:45,196 DEBUG [StoreCloser-TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/b437db3e2d7744c58757ce0b897864b0 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/b437db3e2d7744c58757ce0b897864b0 2024-11-26T19:14:45,197 DEBUG [StoreCloser-TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/8c028ce2f77d42a09b395964a2e95be0 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/8c028ce2f77d42a09b395964a2e95be0 2024-11-26T19:14:45,199 DEBUG [StoreCloser-TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/fc0a8189af804701893b092b2dbe9395 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/fc0a8189af804701893b092b2dbe9395 2024-11-26T19:14:45,201 DEBUG [StoreCloser-TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/ad8bcceb72fe4a1a8b85bc1fcab0ebf7, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/38dbac8e3b294b3da557f07d311f730d, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/4296d4424a504e21a4a50ce8beba0baa, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/3b6bf4ae7c4b4988b33edda9ed1fe305, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/bd5cacf6baaf429fb733a8a6f61e8711, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/ddcadf238f7b44328b5f67863ed8b5db, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/979c1f97799c400a851a925b214d4b7f, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/e8bfc2a4d27f4755b91f11dc8f98d904, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/ab08769c3ac146fd9c5e36d6e078de72, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/e8788ae520b84ff9aff0be6675c1c5b8, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/cabffbcc40e1472f998cec0d94529e4a, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/de440b23eb3e4a6e906face8234ac12c, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/7d9ce28c36d2421f8134b71ca2e95216, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/009b75cfad224716865d1635886facf3, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/69dec79ce76a43199606362ff99df531, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/40ebf4ff13404ea0b4f841745cd5421f, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/fd349611854a47aab9acdf3ef75aff32, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/96e1560c0b164ff5bcc212fd25486acf, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/8dc74111e6c3484abb5ce11d12bbd4ae, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/0759c100aa7845578cd0ae42bfe7de2d, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/5e159984f99c41a8b4f1885cbb877ee2, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/e28e2c5d76ae4c0593cb903a0ec2075a, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/6d0349a759884b6b8e57d6420b7594cf, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/290007cee1454f9fa36fa5a9062f90d9, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/47f399bfc1b84f7d8e5f55c5073fbec3, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/1016ccddb6d94ca4aaa8f83292074fd3, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/ec2130b9fd5c4c4f9ebf8223d8492a33, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/5173fca9e04d469e9814c5043a47b8ed, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/01a7489108b6460fac0e21dbd14e4b5b, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/64d77f8bc3174571ad7dadfef44c591f, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/89bcd3e7a1454002b2b9b001bd940ee5] to archive 2024-11-26T19:14:45,202 DEBUG [StoreCloser-TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-26T19:14:45,203 DEBUG [StoreCloser-TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/ad8bcceb72fe4a1a8b85bc1fcab0ebf7 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/ad8bcceb72fe4a1a8b85bc1fcab0ebf7 2024-11-26T19:14:45,204 DEBUG [StoreCloser-TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/38dbac8e3b294b3da557f07d311f730d to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/38dbac8e3b294b3da557f07d311f730d 2024-11-26T19:14:45,205 DEBUG [StoreCloser-TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/4296d4424a504e21a4a50ce8beba0baa to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/4296d4424a504e21a4a50ce8beba0baa 2024-11-26T19:14:45,206 DEBUG [StoreCloser-TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/3b6bf4ae7c4b4988b33edda9ed1fe305 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/3b6bf4ae7c4b4988b33edda9ed1fe305 2024-11-26T19:14:45,207 DEBUG [StoreCloser-TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/bd5cacf6baaf429fb733a8a6f61e8711 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/bd5cacf6baaf429fb733a8a6f61e8711 2024-11-26T19:14:45,208 DEBUG [StoreCloser-TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/ddcadf238f7b44328b5f67863ed8b5db to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/ddcadf238f7b44328b5f67863ed8b5db 2024-11-26T19:14:45,209 DEBUG [StoreCloser-TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/979c1f97799c400a851a925b214d4b7f to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/979c1f97799c400a851a925b214d4b7f 2024-11-26T19:14:45,209 DEBUG [StoreCloser-TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/e8bfc2a4d27f4755b91f11dc8f98d904 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/e8bfc2a4d27f4755b91f11dc8f98d904 2024-11-26T19:14:45,210 DEBUG [StoreCloser-TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/ab08769c3ac146fd9c5e36d6e078de72 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/ab08769c3ac146fd9c5e36d6e078de72 2024-11-26T19:14:45,211 DEBUG [StoreCloser-TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/e8788ae520b84ff9aff0be6675c1c5b8 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/e8788ae520b84ff9aff0be6675c1c5b8 2024-11-26T19:14:45,212 DEBUG [StoreCloser-TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/cabffbcc40e1472f998cec0d94529e4a to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/cabffbcc40e1472f998cec0d94529e4a 2024-11-26T19:14:45,213 DEBUG [StoreCloser-TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/de440b23eb3e4a6e906face8234ac12c to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/de440b23eb3e4a6e906face8234ac12c 2024-11-26T19:14:45,214 DEBUG [StoreCloser-TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/7d9ce28c36d2421f8134b71ca2e95216 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/7d9ce28c36d2421f8134b71ca2e95216 2024-11-26T19:14:45,215 DEBUG [StoreCloser-TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/009b75cfad224716865d1635886facf3 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/009b75cfad224716865d1635886facf3 2024-11-26T19:14:45,216 DEBUG [StoreCloser-TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/69dec79ce76a43199606362ff99df531 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/69dec79ce76a43199606362ff99df531 2024-11-26T19:14:45,217 DEBUG [StoreCloser-TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/40ebf4ff13404ea0b4f841745cd5421f to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/40ebf4ff13404ea0b4f841745cd5421f 2024-11-26T19:14:45,218 DEBUG [StoreCloser-TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/fd349611854a47aab9acdf3ef75aff32 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/fd349611854a47aab9acdf3ef75aff32 2024-11-26T19:14:45,219 DEBUG [StoreCloser-TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/96e1560c0b164ff5bcc212fd25486acf to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/96e1560c0b164ff5bcc212fd25486acf 2024-11-26T19:14:45,219 DEBUG [StoreCloser-TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/8dc74111e6c3484abb5ce11d12bbd4ae to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/8dc74111e6c3484abb5ce11d12bbd4ae 2024-11-26T19:14:45,220 DEBUG [StoreCloser-TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/0759c100aa7845578cd0ae42bfe7de2d to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/0759c100aa7845578cd0ae42bfe7de2d 2024-11-26T19:14:45,221 DEBUG [StoreCloser-TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/5e159984f99c41a8b4f1885cbb877ee2 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/5e159984f99c41a8b4f1885cbb877ee2 2024-11-26T19:14:45,222 DEBUG [StoreCloser-TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/e28e2c5d76ae4c0593cb903a0ec2075a to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/e28e2c5d76ae4c0593cb903a0ec2075a 2024-11-26T19:14:45,223 DEBUG [StoreCloser-TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/6d0349a759884b6b8e57d6420b7594cf to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/6d0349a759884b6b8e57d6420b7594cf 2024-11-26T19:14:45,224 DEBUG [StoreCloser-TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/290007cee1454f9fa36fa5a9062f90d9 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/290007cee1454f9fa36fa5a9062f90d9 2024-11-26T19:14:45,225 DEBUG [StoreCloser-TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/47f399bfc1b84f7d8e5f55c5073fbec3 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/47f399bfc1b84f7d8e5f55c5073fbec3 2024-11-26T19:14:45,226 DEBUG [StoreCloser-TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/1016ccddb6d94ca4aaa8f83292074fd3 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/1016ccddb6d94ca4aaa8f83292074fd3 2024-11-26T19:14:45,227 DEBUG [StoreCloser-TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/ec2130b9fd5c4c4f9ebf8223d8492a33 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/ec2130b9fd5c4c4f9ebf8223d8492a33 2024-11-26T19:14:45,228 DEBUG [StoreCloser-TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/5173fca9e04d469e9814c5043a47b8ed to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/5173fca9e04d469e9814c5043a47b8ed 2024-11-26T19:14:45,229 DEBUG [StoreCloser-TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/01a7489108b6460fac0e21dbd14e4b5b to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/01a7489108b6460fac0e21dbd14e4b5b 2024-11-26T19:14:45,230 DEBUG [StoreCloser-TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/64d77f8bc3174571ad7dadfef44c591f to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/64d77f8bc3174571ad7dadfef44c591f 2024-11-26T19:14:45,231 DEBUG [StoreCloser-TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/89bcd3e7a1454002b2b9b001bd940ee5 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/89bcd3e7a1454002b2b9b001bd940ee5 2024-11-26T19:14:45,233 DEBUG [StoreCloser-TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/8f94eb4760c445b3825fc75efb49f2e0, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/9dad166e46f84480a73d811efbede772, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/f8158a46268440118a218d19eddc5eee, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/d65e62fe075341f383973364f61ba044, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/af6d53adee09486cb3a9c0fcf414330e, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/3e926e1f27b543698fc424bc4c4747b5, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/adcb77837cf541cf93981581ea852170, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/37a5a3dec23f4be88fb387744c79f1f2, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/5d07cf503cc1413bac0ecef9604f94bb, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/0386ac12602144ff9051f9a2b028b24e, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/231fd54ea85241b69dc73d8e4ed786ab, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/d199d3678a9a4e7bbb84f40c070d22ef, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/2957ce24dd854bf0906556c76e7fa025, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/e5d2a4534a834c0b964fdb830090962a, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/d1145a17dfc64bed940314e9924e86e6, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/6ce2b385a67945b1bb95fbd0ba51abf1, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/98dcb5015a0444e59511871e6b221032, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/a97470276177497c951a479e55ec275a, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/5a80909cae87476ba8b94d49331429cb, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/51524173443d410f9c7e68ae1961096a, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/52d6badf775648aba87302a9eb5c2b6a, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/4b1500fa97fb40d591b01f2a73ceed34, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/c80bc1340b25432cbbb5d53334c161d9, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/9ac2ea240ffd488c9952c3579be4e097, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/c238a51298e445a7bd70307b32e97eb5, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/fcc5e451cdc04b12afd59db7f94992b2, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/a656104fc3bb47bfa0cc3a1b0a687a25, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/60787e1bebea4a948f6a993bbb262299, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/a5cfa0e2a0db42fe924f3ff1a21add2c, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/bdc9a277f358493185cd752eeec97c4e, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/e00cb2156de043b1b5695f2f97c9b8b6] to archive 2024-11-26T19:14:45,234 DEBUG [StoreCloser-TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-26T19:14:45,235 DEBUG [StoreCloser-TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/8f94eb4760c445b3825fc75efb49f2e0 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/8f94eb4760c445b3825fc75efb49f2e0 2024-11-26T19:14:45,237 DEBUG [StoreCloser-TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/9dad166e46f84480a73d811efbede772 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/9dad166e46f84480a73d811efbede772 2024-11-26T19:14:45,238 DEBUG [StoreCloser-TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/f8158a46268440118a218d19eddc5eee to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/f8158a46268440118a218d19eddc5eee 2024-11-26T19:14:45,239 DEBUG [StoreCloser-TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/d65e62fe075341f383973364f61ba044 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/d65e62fe075341f383973364f61ba044 2024-11-26T19:14:45,241 DEBUG [StoreCloser-TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/af6d53adee09486cb3a9c0fcf414330e to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/af6d53adee09486cb3a9c0fcf414330e 2024-11-26T19:14:45,242 DEBUG [StoreCloser-TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/3e926e1f27b543698fc424bc4c4747b5 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/3e926e1f27b543698fc424bc4c4747b5 2024-11-26T19:14:45,243 DEBUG [StoreCloser-TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/adcb77837cf541cf93981581ea852170 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/adcb77837cf541cf93981581ea852170 2024-11-26T19:14:45,244 DEBUG [StoreCloser-TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/37a5a3dec23f4be88fb387744c79f1f2 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/37a5a3dec23f4be88fb387744c79f1f2 2024-11-26T19:14:45,245 DEBUG [StoreCloser-TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/5d07cf503cc1413bac0ecef9604f94bb to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/5d07cf503cc1413bac0ecef9604f94bb 2024-11-26T19:14:45,246 DEBUG [StoreCloser-TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/0386ac12602144ff9051f9a2b028b24e to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/0386ac12602144ff9051f9a2b028b24e 2024-11-26T19:14:45,247 DEBUG [StoreCloser-TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/231fd54ea85241b69dc73d8e4ed786ab to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/231fd54ea85241b69dc73d8e4ed786ab 2024-11-26T19:14:45,248 DEBUG [StoreCloser-TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/d199d3678a9a4e7bbb84f40c070d22ef to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/d199d3678a9a4e7bbb84f40c070d22ef 2024-11-26T19:14:45,249 DEBUG [StoreCloser-TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/2957ce24dd854bf0906556c76e7fa025 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/2957ce24dd854bf0906556c76e7fa025 2024-11-26T19:14:45,262 DEBUG [StoreCloser-TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/e5d2a4534a834c0b964fdb830090962a to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/e5d2a4534a834c0b964fdb830090962a 2024-11-26T19:14:45,263 DEBUG [StoreCloser-TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/d1145a17dfc64bed940314e9924e86e6 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/d1145a17dfc64bed940314e9924e86e6 2024-11-26T19:14:45,265 DEBUG [StoreCloser-TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/6ce2b385a67945b1bb95fbd0ba51abf1 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/6ce2b385a67945b1bb95fbd0ba51abf1 2024-11-26T19:14:45,266 DEBUG [StoreCloser-TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/98dcb5015a0444e59511871e6b221032 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/98dcb5015a0444e59511871e6b221032 2024-11-26T19:14:45,268 DEBUG [StoreCloser-TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/a97470276177497c951a479e55ec275a to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/a97470276177497c951a479e55ec275a 2024-11-26T19:14:45,269 DEBUG [StoreCloser-TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/5a80909cae87476ba8b94d49331429cb to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/5a80909cae87476ba8b94d49331429cb 2024-11-26T19:14:45,270 DEBUG [StoreCloser-TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/51524173443d410f9c7e68ae1961096a to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/51524173443d410f9c7e68ae1961096a 2024-11-26T19:14:45,271 DEBUG [StoreCloser-TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/52d6badf775648aba87302a9eb5c2b6a to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/52d6badf775648aba87302a9eb5c2b6a 2024-11-26T19:14:45,273 DEBUG [StoreCloser-TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/4b1500fa97fb40d591b01f2a73ceed34 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/4b1500fa97fb40d591b01f2a73ceed34 2024-11-26T19:14:45,274 DEBUG [StoreCloser-TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/c80bc1340b25432cbbb5d53334c161d9 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/c80bc1340b25432cbbb5d53334c161d9 2024-11-26T19:14:45,275 DEBUG [StoreCloser-TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/9ac2ea240ffd488c9952c3579be4e097 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/9ac2ea240ffd488c9952c3579be4e097 2024-11-26T19:14:45,276 DEBUG [StoreCloser-TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/c238a51298e445a7bd70307b32e97eb5 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/c238a51298e445a7bd70307b32e97eb5 2024-11-26T19:14:45,277 DEBUG [StoreCloser-TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/fcc5e451cdc04b12afd59db7f94992b2 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/fcc5e451cdc04b12afd59db7f94992b2 2024-11-26T19:14:45,278 DEBUG [StoreCloser-TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/a656104fc3bb47bfa0cc3a1b0a687a25 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/a656104fc3bb47bfa0cc3a1b0a687a25 2024-11-26T19:14:45,279 DEBUG [StoreCloser-TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/60787e1bebea4a948f6a993bbb262299 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/60787e1bebea4a948f6a993bbb262299 2024-11-26T19:14:45,281 DEBUG [StoreCloser-TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/a5cfa0e2a0db42fe924f3ff1a21add2c to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/a5cfa0e2a0db42fe924f3ff1a21add2c 2024-11-26T19:14:45,282 DEBUG [StoreCloser-TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/bdc9a277f358493185cd752eeec97c4e to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/bdc9a277f358493185cd752eeec97c4e 2024-11-26T19:14:45,283 DEBUG [StoreCloser-TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/e00cb2156de043b1b5695f2f97c9b8b6 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/e00cb2156de043b1b5695f2f97c9b8b6 2024-11-26T19:14:45,289 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/recovered.edits/427.seqid, newMaxSeqId=427, maxSeqId=4 2024-11-26T19:14:45,290 INFO [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. 2024-11-26T19:14:45,290 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1676): Region close journal for d23b79ada6d885c09c429afcb7484518: Waiting for close lock at 1732648483083Running coprocessor pre-close hooks at 1732648483083Disabling compacts and flushes for region at 1732648483083Disabling writes for close at 1732648483475 (+392 ms)Obtaining lock to block concurrent updates at 1732648483475Preparing flush snapshotting stores in d23b79ada6d885c09c429afcb7484518 at 1732648483475Finished memstore snapshotting TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1732648483476 (+1 ms)Flushing stores of TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518. at 1732648483476Flushing d23b79ada6d885c09c429afcb7484518/A: creating writer at 1732648483476Flushing mob file d23b79ada6d885c09c429afcb7484518/A: appending metadata at 1732648483488 (+12 ms)Flushing mob file d23b79ada6d885c09c429afcb7484518/A: closing flushed file at 1732648483488Flushing d23b79ada6d885c09c429afcb7484518/A: appending metadata at 1732648483907 (+419 ms)Flushing d23b79ada6d885c09c429afcb7484518/A: closing flushed file at 1732648483907Flushing d23b79ada6d885c09c429afcb7484518/B: creating writer at 1732648484316 (+409 ms)Flushing d23b79ada6d885c09c429afcb7484518/B: appending metadata at 1732648484319 (+3 ms)Flushing d23b79ada6d885c09c429afcb7484518/B: closing flushed file at 1732648484319Flushing d23b79ada6d885c09c429afcb7484518/C: creating writer at 1732648484733 (+414 ms)Flushing d23b79ada6d885c09c429afcb7484518/C: appending metadata at 1732648484736 (+3 ms)Flushing d23b79ada6d885c09c429afcb7484518/C: closing flushed file at 1732648484736Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@407eed0c: reopening flushed file at 1732648485144 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@179a9c6e: reopening flushed file at 1732648485149 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@348ffd51: reopening flushed file at 1732648485153 (+4 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for d23b79ada6d885c09c429afcb7484518 in 1684ms, sequenceid=424, compaction requested=false at 1732648485159 (+6 ms)Writing region close event to WAL at 1732648485285 (+126 ms)Running coprocessor post-close hooks at 1732648485290 (+5 ms)Closed at 1732648485290 2024-11-26T19:14:45,293 INFO [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] handler.UnassignRegionHandler(157): Closed d23b79ada6d885c09c429afcb7484518 2024-11-26T19:14:45,293 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=54 updating hbase:meta row=d23b79ada6d885c09c429afcb7484518, regionState=CLOSED 2024-11-26T19:14:45,295 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=55, ppid=54, state=RUNNABLE, hasLock=false; CloseRegionProcedure d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 because future has completed 2024-11-26T19:14:45,298 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-26T19:14:45,298 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; CloseRegionProcedure d23b79ada6d885c09c429afcb7484518, server=fe76acf732b2,42785,1732648423970 in 2.3660 sec 2024-11-26T19:14:45,300 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=54, resume processing ppid=53 2024-11-26T19:14:45,300 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, ppid=53, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d23b79ada6d885c09c429afcb7484518, UNASSIGN in 2.3720 sec 2024-11-26T19:14:45,302 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-26T19:14:45,302 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.3760 sec 2024-11-26T19:14:45,304 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732648485304"}]},"ts":"1732648485304"} 2024-11-26T19:14:45,306 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-26T19:14:45,306 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-26T19:14:45,308 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.3930 sec 2024-11-26T19:14:47,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-26T19:14:47,082 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-26T19:14:47,082 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-26T19:14:47,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T19:14:47,084 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=56, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T19:14:47,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-26T19:14:47,085 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=56, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T19:14:47,088 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518 2024-11-26T19:14:47,091 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A, FileablePath, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B, FileablePath, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C, FileablePath, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/recovered.edits] 2024-11-26T19:14:47,094 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/27b103e1563e4e5585059fd805f66860 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/27b103e1563e4e5585059fd805f66860 2024-11-26T19:14:47,095 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/824e302f866b45a9b6edbed9ce760038 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/A/824e302f866b45a9b6edbed9ce760038 2024-11-26T19:14:47,098 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/b450554817554703bf4eaf50688ed8f7 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/b450554817554703bf4eaf50688ed8f7 2024-11-26T19:14:47,099 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/f7dc3bd6baca4d4681e8631e4249df42 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/B/f7dc3bd6baca4d4681e8631e4249df42 2024-11-26T19:14:47,102 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/81f58b99d3144cdc9b501bc4583e4bf6 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/81f58b99d3144cdc9b501bc4583e4bf6 2024-11-26T19:14:47,103 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/e310ede3e8414ebcabab58397d8984eb to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/C/e310ede3e8414ebcabab58397d8984eb 2024-11-26T19:14:47,106 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/recovered.edits/427.seqid to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518/recovered.edits/427.seqid 2024-11-26T19:14:47,107 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/d23b79ada6d885c09c429afcb7484518 2024-11-26T19:14:47,107 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-26T19:14:47,107 DEBUG [PEWorker-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-26T19:14:47,108 DEBUG [PEWorker-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-26T19:14:47,113 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126093bea2423844ad188d810ca49f86b49_d23b79ada6d885c09c429afcb7484518 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126093bea2423844ad188d810ca49f86b49_d23b79ada6d885c09c429afcb7484518 2024-11-26T19:14:47,114 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411260a5ccdabbbbb41db926574b7c019f129_d23b79ada6d885c09c429afcb7484518 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411260a5ccdabbbbb41db926574b7c019f129_d23b79ada6d885c09c429afcb7484518 2024-11-26T19:14:47,115 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411260e6b11aed79648af8be6dcae3a9e4d5b_d23b79ada6d885c09c429afcb7484518 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411260e6b11aed79648af8be6dcae3a9e4d5b_d23b79ada6d885c09c429afcb7484518 2024-11-26T19:14:47,116 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126270dd33e37ee46e8b906d207d2b1c347_d23b79ada6d885c09c429afcb7484518 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126270dd33e37ee46e8b906d207d2b1c347_d23b79ada6d885c09c429afcb7484518 2024-11-26T19:14:47,117 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126292ac7f09cbc4d7fa8638298acc199d2_d23b79ada6d885c09c429afcb7484518 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126292ac7f09cbc4d7fa8638298acc199d2_d23b79ada6d885c09c429afcb7484518 2024-11-26T19:14:47,118 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411263a1a9e908c5e4b91847310fac402566e_d23b79ada6d885c09c429afcb7484518 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411263a1a9e908c5e4b91847310fac402566e_d23b79ada6d885c09c429afcb7484518 2024-11-26T19:14:47,119 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411265b11e856cd734b669de56eb444a54589_d23b79ada6d885c09c429afcb7484518 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411265b11e856cd734b669de56eb444a54589_d23b79ada6d885c09c429afcb7484518 2024-11-26T19:14:47,121 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411265f50b4af36cf4beab4ed28f4233550ad_d23b79ada6d885c09c429afcb7484518 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411265f50b4af36cf4beab4ed28f4233550ad_d23b79ada6d885c09c429afcb7484518 2024-11-26T19:14:47,122 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126704cc4aefb3743abb8bb4de6b79b480d_d23b79ada6d885c09c429afcb7484518 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126704cc4aefb3743abb8bb4de6b79b480d_d23b79ada6d885c09c429afcb7484518 2024-11-26T19:14:47,123 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112671f0df6a11394169835eceb14e713aef_d23b79ada6d885c09c429afcb7484518 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112671f0df6a11394169835eceb14e713aef_d23b79ada6d885c09c429afcb7484518 2024-11-26T19:14:47,124 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411267f9c22821b024c02bd8b82676d5c5585_d23b79ada6d885c09c429afcb7484518 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411267f9c22821b024c02bd8b82676d5c5585_d23b79ada6d885c09c429afcb7484518 2024-11-26T19:14:47,125 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112688f59e068c5e4e1a95478a99d2582198_d23b79ada6d885c09c429afcb7484518 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112688f59e068c5e4e1a95478a99d2582198_d23b79ada6d885c09c429afcb7484518 2024-11-26T19:14:47,128 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126931396bb95f8476eb7f2f7aef7d53692_d23b79ada6d885c09c429afcb7484518 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126931396bb95f8476eb7f2f7aef7d53692_d23b79ada6d885c09c429afcb7484518 2024-11-26T19:14:47,129 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126971e46746155449d8509f206585fb225_d23b79ada6d885c09c429afcb7484518 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126971e46746155449d8509f206585fb225_d23b79ada6d885c09c429afcb7484518 2024-11-26T19:14:47,130 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411269d17788213414966877eab02ed2f6192_d23b79ada6d885c09c429afcb7484518 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411269d17788213414966877eab02ed2f6192_d23b79ada6d885c09c429afcb7484518 2024-11-26T19:14:47,132 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126a94e70e4714d4488b125023f21878042_d23b79ada6d885c09c429afcb7484518 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126a94e70e4714d4488b125023f21878042_d23b79ada6d885c09c429afcb7484518 2024-11-26T19:14:47,133 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126b60e6ff4e1794913840a3fc9346f6aaf_d23b79ada6d885c09c429afcb7484518 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126b60e6ff4e1794913840a3fc9346f6aaf_d23b79ada6d885c09c429afcb7484518 2024-11-26T19:14:47,134 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126c994d2954d5e412894a8303b6a815858_d23b79ada6d885c09c429afcb7484518 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126c994d2954d5e412894a8303b6a815858_d23b79ada6d885c09c429afcb7484518 2024-11-26T19:14:47,135 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126e189e8991d33425fad185dfba3b59ef3_d23b79ada6d885c09c429afcb7484518 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126e189e8991d33425fad185dfba3b59ef3_d23b79ada6d885c09c429afcb7484518 2024-11-26T19:14:47,137 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126e1e0e6beeea34ff19f545205665625ad_d23b79ada6d885c09c429afcb7484518 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126e1e0e6beeea34ff19f545205665625ad_d23b79ada6d885c09c429afcb7484518 2024-11-26T19:14:47,139 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126ec4feebfe1e148e1bce402ff2dd94e3d_d23b79ada6d885c09c429afcb7484518 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126ec4feebfe1e148e1bce402ff2dd94e3d_d23b79ada6d885c09c429afcb7484518 2024-11-26T19:14:47,140 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126f50be5b5415343a686434b851ff20c66_d23b79ada6d885c09c429afcb7484518 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126f50be5b5415343a686434b851ff20c66_d23b79ada6d885c09c429afcb7484518 2024-11-26T19:14:47,141 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126fd41d313179546078f060d1c132ba1bc_d23b79ada6d885c09c429afcb7484518 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126fd41d313179546078f060d1c132ba1bc_d23b79ada6d885c09c429afcb7484518 2024-11-26T19:14:47,142 DEBUG [PEWorker-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-26T19:14:47,144 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=56, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T19:14:47,146 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-26T19:14:47,149 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-26T19:14:47,150 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=56, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T19:14:47,150 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-26T19:14:47,150 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732648487150"}]},"ts":"9223372036854775807"} 2024-11-26T19:14:47,152 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-26T19:14:47,152 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => d23b79ada6d885c09c429afcb7484518, NAME => 'TestAcidGuarantees,,1732648456493.d23b79ada6d885c09c429afcb7484518.', STARTKEY => '', ENDKEY => ''}] 2024-11-26T19:14:47,152 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-26T19:14:47,152 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732648487152"}]},"ts":"9223372036854775807"} 2024-11-26T19:14:47,154 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-26T19:14:47,155 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=56, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T19:14:47,156 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 73 msec 2024-11-26T19:14:47,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-26T19:14:47,192 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-26T19:14:47,192 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-26T19:14:47,202 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=251 (was 248) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/efb451b3-ba8c-40d2-3b0c-db6f88deea36/cluster_7d054ae1-c0e9-960d-2aea-ddd34a033944/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1480762926_22 at /127.0.0.1:34290 [Waiting for operation #301] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1480762926_22 at /127.0.0.1:44662 [Waiting for operation #257] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1331211566_22 at /127.0.0.1:46944 [Waiting for operation #663] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/efb451b3-ba8c-40d2-3b0c-db6f88deea36/cluster_7d054ae1-c0e9-960d-2aea-ddd34a033944/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1331211566_22 at /127.0.0.1:44800 [Waiting for operation #199] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=459 (was 453) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=483 (was 448) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=6597 (was 6953) 2024-11-26T19:14:47,210 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=251, OpenFileDescriptor=459, MaxFileDescriptor=1048576, SystemLoadAverage=483, ProcessCount=11, AvailableMemoryMB=6596 2024-11-26T19:14:47,211 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-26T19:14:47,212 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-26T19:14:47,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] procedure2.ProcedureExecutor(1139): Stored pid=57, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-26T19:14:47,214 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-26T19:14:47,214 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:47,214 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 57 2024-11-26T19:14:47,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=57 2024-11-26T19:14:47,215 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-26T19:14:47,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742088_1264 (size=1061) 2024-11-26T19:14:47,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=57 2024-11-26T19:14:47,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=57 2024-11-26T19:14:47,622 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16 2024-11-26T19:14:47,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742089_1265 (size=53) 2024-11-26T19:14:47,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=57 2024-11-26T19:14:48,028 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T19:14:48,028 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 9fea48dabba44dd37de365711fba859f, disabling compactions & flushes 2024-11-26T19:14:48,028 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. 2024-11-26T19:14:48,028 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. 2024-11-26T19:14:48,028 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. after waiting 0 ms 2024-11-26T19:14:48,028 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. 2024-11-26T19:14:48,028 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. 2024-11-26T19:14:48,028 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 9fea48dabba44dd37de365711fba859f: Waiting for close lock at 1732648488028Disabling compacts and flushes for region at 1732648488028Disabling writes for close at 1732648488028Writing region close event to WAL at 1732648488028Closed at 1732648488028 2024-11-26T19:14:48,029 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-26T19:14:48,030 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732648488029"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732648488029"}]},"ts":"1732648488029"} 2024-11-26T19:14:48,031 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-26T19:14:48,032 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-26T19:14:48,032 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732648488032"}]},"ts":"1732648488032"} 2024-11-26T19:14:48,033 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-26T19:14:48,034 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9fea48dabba44dd37de365711fba859f, ASSIGN}] 2024-11-26T19:14:48,035 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9fea48dabba44dd37de365711fba859f, ASSIGN 2024-11-26T19:14:48,035 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=9fea48dabba44dd37de365711fba859f, ASSIGN; state=OFFLINE, location=fe76acf732b2,42785,1732648423970; forceNewPlan=false, retain=false 2024-11-26T19:14:48,186 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=58 updating hbase:meta row=9fea48dabba44dd37de365711fba859f, regionState=OPENING, regionLocation=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:48,188 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=9fea48dabba44dd37de365711fba859f, ASSIGN because future has completed 2024-11-26T19:14:48,188 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970}] 2024-11-26T19:14:48,344 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. 2024-11-26T19:14:48,344 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(7752): Opening region: {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} 2024-11-26T19:14:48,345 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 9fea48dabba44dd37de365711fba859f 2024-11-26T19:14:48,345 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T19:14:48,345 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(7794): checking encryption for 9fea48dabba44dd37de365711fba859f 2024-11-26T19:14:48,345 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(7797): checking classloading for 9fea48dabba44dd37de365711fba859f 2024-11-26T19:14:48,348 INFO [StoreOpener-9fea48dabba44dd37de365711fba859f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 9fea48dabba44dd37de365711fba859f 2024-11-26T19:14:48,349 INFO [StoreOpener-9fea48dabba44dd37de365711fba859f-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-26T19:14:48,350 INFO [StoreOpener-9fea48dabba44dd37de365711fba859f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9fea48dabba44dd37de365711fba859f columnFamilyName A 2024-11-26T19:14:48,350 DEBUG [StoreOpener-9fea48dabba44dd37de365711fba859f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:48,350 INFO [StoreOpener-9fea48dabba44dd37de365711fba859f-1 {}] regionserver.HStore(327): Store=9fea48dabba44dd37de365711fba859f/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T19:14:48,350 INFO [StoreOpener-9fea48dabba44dd37de365711fba859f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 9fea48dabba44dd37de365711fba859f 2024-11-26T19:14:48,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=57 2024-11-26T19:14:48,351 INFO [StoreOpener-9fea48dabba44dd37de365711fba859f-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-26T19:14:48,351 INFO [StoreOpener-9fea48dabba44dd37de365711fba859f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9fea48dabba44dd37de365711fba859f columnFamilyName B 2024-11-26T19:14:48,351 DEBUG [StoreOpener-9fea48dabba44dd37de365711fba859f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:48,352 INFO [StoreOpener-9fea48dabba44dd37de365711fba859f-1 {}] regionserver.HStore(327): Store=9fea48dabba44dd37de365711fba859f/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T19:14:48,352 INFO [StoreOpener-9fea48dabba44dd37de365711fba859f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 9fea48dabba44dd37de365711fba859f 2024-11-26T19:14:48,353 INFO [StoreOpener-9fea48dabba44dd37de365711fba859f-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-26T19:14:48,353 INFO [StoreOpener-9fea48dabba44dd37de365711fba859f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9fea48dabba44dd37de365711fba859f columnFamilyName C 2024-11-26T19:14:48,353 DEBUG [StoreOpener-9fea48dabba44dd37de365711fba859f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:48,353 INFO [StoreOpener-9fea48dabba44dd37de365711fba859f-1 {}] regionserver.HStore(327): Store=9fea48dabba44dd37de365711fba859f/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T19:14:48,353 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. 2024-11-26T19:14:48,354 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1038): replaying wal for 9fea48dabba44dd37de365711fba859f 2024-11-26T19:14:48,354 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f 2024-11-26T19:14:48,355 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f 2024-11-26T19:14:48,356 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1048): stopping wal replay for 9fea48dabba44dd37de365711fba859f 2024-11-26T19:14:48,356 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1060): Cleaning up temporary data for 9fea48dabba44dd37de365711fba859f 2024-11-26T19:14:48,357 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-26T19:14:48,358 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1093): writing seq id for 9fea48dabba44dd37de365711fba859f 2024-11-26T19:14:48,359 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T19:14:48,360 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1114): Opened 9fea48dabba44dd37de365711fba859f; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70080877, jitterRate=0.044286444783210754}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-26T19:14:48,360 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 9fea48dabba44dd37de365711fba859f 2024-11-26T19:14:48,360 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1006): Region open journal for 9fea48dabba44dd37de365711fba859f: Running coprocessor pre-open hook at 1732648488345Writing region info on filesystem at 1732648488345Initializing all the Stores at 1732648488346 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732648488346Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732648488348 (+2 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732648488348Cleaning up temporary data from old regions at 1732648488356 (+8 ms)Running coprocessor post-open hooks at 1732648488360 (+4 ms)Region opened successfully at 1732648488360 2024-11-26T19:14:48,361 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., pid=59, masterSystemTime=1732648488340 2024-11-26T19:14:48,362 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. 2024-11-26T19:14:48,362 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. 2024-11-26T19:14:48,363 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=58 updating hbase:meta row=9fea48dabba44dd37de365711fba859f, regionState=OPEN, openSeqNum=2, regionLocation=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:48,365 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=59, ppid=58, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 because future has completed 2024-11-26T19:14:48,367 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-26T19:14:48,367 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; OpenRegionProcedure 9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 in 177 msec 2024-11-26T19:14:48,370 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=58, resume processing ppid=57 2024-11-26T19:14:48,370 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9fea48dabba44dd37de365711fba859f, ASSIGN in 333 msec 2024-11-26T19:14:48,371 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-26T19:14:48,371 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732648488371"}]},"ts":"1732648488371"} 2024-11-26T19:14:48,373 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-26T19:14:48,374 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-26T19:14:48,376 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1620 sec 2024-11-26T19:14:49,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=57 2024-11-26T19:14:49,362 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-26T19:14:49,363 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ce28163, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:14:49,363 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fe76acf732b2,33811,-1 for getting cluster id 2024-11-26T19:14:49,363 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T19:14:49,364 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3cec4e24-71db-4cd0-b63a-4ea30a9b160b' 2024-11-26T19:14:49,365 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T19:14:49,365 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3cec4e24-71db-4cd0-b63a-4ea30a9b160b" 2024-11-26T19:14:49,365 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58c6d79f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:14:49,365 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fe76acf732b2,33811,-1] 2024-11-26T19:14:49,365 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T19:14:49,365 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:14:49,366 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49682, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T19:14:49,367 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55d590e9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:14:49,367 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T19:14:49,368 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fe76acf732b2,42785,1732648423970, seqNum=-1] 2024-11-26T19:14:49,368 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T19:14:49,369 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53554, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T19:14:49,370 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-26T19:14:49,370 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is fe76acf732b2,33811,1732648423099 2024-11-26T19:14:49,370 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@2bfa211e 2024-11-26T19:14:49,370 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-26T19:14:49,371 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49690, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-26T19:14:49,372 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7642a08c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:14:49,372 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fe76acf732b2,33811,-1 for getting cluster id 2024-11-26T19:14:49,372 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T19:14:49,373 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3cec4e24-71db-4cd0-b63a-4ea30a9b160b' 2024-11-26T19:14:49,373 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T19:14:49,373 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3cec4e24-71db-4cd0-b63a-4ea30a9b160b" 2024-11-26T19:14:49,373 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@779aea0f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:14:49,373 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fe76acf732b2,33811,-1] 2024-11-26T19:14:49,373 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T19:14:49,373 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:14:49,374 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49718, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T19:14:49,374 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@370f7864, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:14:49,375 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ecfb794, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:14:49,375 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fe76acf732b2,33811,-1 for getting cluster id 2024-11-26T19:14:49,375 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T19:14:49,376 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3cec4e24-71db-4cd0-b63a-4ea30a9b160b' 2024-11-26T19:14:49,376 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T19:14:49,376 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3cec4e24-71db-4cd0-b63a-4ea30a9b160b" 2024-11-26T19:14:49,376 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46d979ac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:14:49,376 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fe76acf732b2,33811,-1] 2024-11-26T19:14:49,376 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T19:14:49,376 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:14:49,377 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49734, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T19:14:49,377 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ff50579, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:14:49,378 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ff0f448, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:14:49,378 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fe76acf732b2,33811,-1 for getting cluster id 2024-11-26T19:14:49,378 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T19:14:49,378 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3cec4e24-71db-4cd0-b63a-4ea30a9b160b' 2024-11-26T19:14:49,378 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T19:14:49,379 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3cec4e24-71db-4cd0-b63a-4ea30a9b160b" 2024-11-26T19:14:49,379 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3738026c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:14:49,379 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fe76acf732b2,33811,-1] 2024-11-26T19:14:49,379 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T19:14:49,379 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:14:49,379 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49758, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T19:14:49,380 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28d204f1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:14:49,380 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d560643, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:14:49,380 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fe76acf732b2,33811,-1 for getting cluster id 2024-11-26T19:14:49,381 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T19:14:49,381 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3cec4e24-71db-4cd0-b63a-4ea30a9b160b' 2024-11-26T19:14:49,381 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T19:14:49,381 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3cec4e24-71db-4cd0-b63a-4ea30a9b160b" 2024-11-26T19:14:49,381 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29635038, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:14:49,381 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fe76acf732b2,33811,-1] 2024-11-26T19:14:49,381 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T19:14:49,382 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:14:49,382 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49766, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T19:14:49,383 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33e5a3de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:14:49,383 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6509bdf5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:14:49,383 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fe76acf732b2,33811,-1 for getting cluster id 2024-11-26T19:14:49,383 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T19:14:49,384 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3cec4e24-71db-4cd0-b63a-4ea30a9b160b' 2024-11-26T19:14:49,384 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T19:14:49,384 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3cec4e24-71db-4cd0-b63a-4ea30a9b160b" 2024-11-26T19:14:49,384 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ec305f2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:14:49,384 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fe76acf732b2,33811,-1] 2024-11-26T19:14:49,385 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T19:14:49,385 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:14:49,385 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49794, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T19:14:49,385 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d0038e1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:14:49,386 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@beb69b9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:14:49,386 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fe76acf732b2,33811,-1 for getting cluster id 2024-11-26T19:14:49,386 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T19:14:49,387 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3cec4e24-71db-4cd0-b63a-4ea30a9b160b' 2024-11-26T19:14:49,387 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T19:14:49,387 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3cec4e24-71db-4cd0-b63a-4ea30a9b160b" 2024-11-26T19:14:49,387 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d2317c9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:14:49,387 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fe76acf732b2,33811,-1] 2024-11-26T19:14:49,387 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T19:14:49,387 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:14:49,388 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49808, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T19:14:49,388 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c64600a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:14:49,389 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6643da6c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:14:49,389 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fe76acf732b2,33811,-1 for getting cluster id 2024-11-26T19:14:49,389 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T19:14:49,389 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3cec4e24-71db-4cd0-b63a-4ea30a9b160b' 2024-11-26T19:14:49,389 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T19:14:49,389 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3cec4e24-71db-4cd0-b63a-4ea30a9b160b" 2024-11-26T19:14:49,390 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3180b6d4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:14:49,390 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fe76acf732b2,33811,-1] 2024-11-26T19:14:49,390 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T19:14:49,390 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:14:49,390 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49826, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T19:14:49,391 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d24d95a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:14:49,391 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4829bcb7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:14:49,391 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fe76acf732b2,33811,-1 for getting cluster id 2024-11-26T19:14:49,391 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T19:14:49,392 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3cec4e24-71db-4cd0-b63a-4ea30a9b160b' 2024-11-26T19:14:49,392 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T19:14:49,392 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3cec4e24-71db-4cd0-b63a-4ea30a9b160b" 2024-11-26T19:14:49,392 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64d509c4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:14:49,392 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fe76acf732b2,33811,-1] 2024-11-26T19:14:49,392 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T19:14:49,392 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:14:49,393 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49844, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T19:14:49,393 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@344ac1c0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:14:49,394 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10d96952, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:14:49,394 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fe76acf732b2,33811,-1 for getting cluster id 2024-11-26T19:14:49,394 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T19:14:49,394 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3cec4e24-71db-4cd0-b63a-4ea30a9b160b' 2024-11-26T19:14:49,395 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T19:14:49,395 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3cec4e24-71db-4cd0-b63a-4ea30a9b160b" 2024-11-26T19:14:49,395 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f4436fe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:14:49,395 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fe76acf732b2,33811,-1] 2024-11-26T19:14:49,395 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T19:14:49,395 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:14:49,395 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49852, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T19:14:49,396 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10a3d105, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:14:49,396 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24595b5e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:14:49,396 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fe76acf732b2,33811,-1 for getting cluster id 2024-11-26T19:14:49,396 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T19:14:49,397 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3cec4e24-71db-4cd0-b63a-4ea30a9b160b' 2024-11-26T19:14:49,397 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T19:14:49,397 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3cec4e24-71db-4cd0-b63a-4ea30a9b160b" 2024-11-26T19:14:49,397 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ee8448a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:14:49,397 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fe76acf732b2,33811,-1] 2024-11-26T19:14:49,397 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T19:14:49,397 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:14:49,398 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49872, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T19:14:49,398 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@662f9b45, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:14:49,399 DEBUG [Thread-1218 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T19:14:49,399 DEBUG [Thread-1226 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T19:14:49,399 DEBUG [Thread-1224 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T19:14:49,399 DEBUG [Thread-1222 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T19:14:49,399 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T19:14:49,399 DEBUG [Thread-1237 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T19:14:49,399 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fe76acf732b2,42785,1732648423970, seqNum=-1] 2024-11-26T19:14:49,399 DEBUG [Thread-1220 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T19:14:49,399 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fe76acf732b2,42785,1732648423970, seqNum=-1] 2024-11-26T19:14:49,399 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fe76acf732b2,42785,1732648423970, seqNum=-1] 2024-11-26T19:14:49,400 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T19:14:49,400 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T19:14:49,400 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T19:14:49,400 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fe76acf732b2,42785,1732648423970, seqNum=-1] 2024-11-26T19:14:49,400 DEBUG [Thread-1231 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T19:14:49,401 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T19:14:49,401 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53558, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T19:14:49,401 DEBUG [Thread-1235 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T19:14:49,401 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53556, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T19:14:49,401 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fe76acf732b2,42785,1732648423970, seqNum=-1] 2024-11-26T19:14:49,401 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fe76acf732b2,42785,1732648423970, seqNum=-1] 2024-11-26T19:14:49,401 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fe76acf732b2,42785,1732648423970, seqNum=-1] 2024-11-26T19:14:49,401 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T19:14:49,402 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T19:14:49,402 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T19:14:49,402 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fe76acf732b2,42785,1732648423970, seqNum=-1] 2024-11-26T19:14:49,402 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T19:14:49,403 DEBUG [Thread-1229 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T19:14:49,403 DEBUG [Thread-1233 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T19:14:49,403 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2] 2024-11-26T19:14:49,403 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2] 2024-11-26T19:14:49,403 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53568, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T19:14:49,403 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fe76acf732b2,42785,1732648423970, seqNum=-1] 2024-11-26T19:14:49,404 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T19:14:49,404 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53604, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T19:14:49,404 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53582, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T19:14:49,404 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53590, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T19:14:49,404 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53608, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T19:14:49,404 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fe76acf732b2,42785,1732648423970, seqNum=-1] 2024-11-26T19:14:49,405 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T19:14:49,405 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53624, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T19:14:49,405 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53632, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T19:14:49,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees 2024-11-26T19:14:49,405 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53638, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T19:14:49,406 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2] 2024-11-26T19:14:49,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-26T19:14:49,407 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2] 2024-11-26T19:14:49,408 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T19:14:49,409 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2] 2024-11-26T19:14:49,409 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2] 2024-11-26T19:14:49,409 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2] 2024-11-26T19:14:49,409 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T19:14:49,410 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T19:14:49,410 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2] 2024-11-26T19:14:49,410 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2] 2024-11-26T19:14:49,411 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2] 2024-11-26T19:14:49,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(8855): Flush requested on 9fea48dabba44dd37de365711fba859f 2024-11-26T19:14:49,417 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9fea48dabba44dd37de365711fba859f 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-26T19:14:49,417 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9fea48dabba44dd37de365711fba859f, store=A 2024-11-26T19:14:49,417 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:14:49,417 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9fea48dabba44dd37de365711fba859f, store=B 2024-11-26T19:14:49,417 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:14:49,417 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9fea48dabba44dd37de365711fba859f, store=C 2024-11-26T19:14:49,418 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:14:49,448 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/A/80ca1aabf1a841c9aadcdd990dd4c53f is 50, key is test_row_0/A:col10/1732648489407/Put/seqid=0 2024-11-26T19:14:49,452 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732648489436, max timestamp=1732648489449Num uniques 859; 2024-11-26T19:14:49,452 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-26T19:14:49,453 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732648489436, max timestamp=1732648489449Num uniques 859; 2024-11-26T19:14:49,453 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-26T19:14:49,455 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732648489436, max timestamp=1732648489449Num uniques 859; 2024-11-26T19:14:49,456 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-26T19:14:49,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742090_1266 (size=14341) 2024-11-26T19:14:49,479 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:49,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53556 deadline: 1732648549472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:49,480 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:49,480 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:49,480 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:49,481 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:49,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53632 deadline: 1732648549474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:49,482 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:49,482 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:49,482 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:49,482 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:49,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53582 deadline: 1732648549476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:49,482 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:49,482 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:49,482 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:49,487 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:49,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53558 deadline: 1732648549479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:49,487 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:49,487 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:49,487 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:49,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53568 deadline: 1732648549481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:49,487 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:49,488 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:49,488 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:49,488 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:49,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-26T19:14:49,563 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-26T19:14:49,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. 2024-11-26T19:14:49,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. as already flushing 2024-11-26T19:14:49,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. 2024-11-26T19:14:49,564 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:49,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:49,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:49,595 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:49,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53556 deadline: 1732648549591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:49,595 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:49,595 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:49,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53632 deadline: 1732648549592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:49,595 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:49,595 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:49,596 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:49,596 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:49,596 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:49,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53582 deadline: 1732648549592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:49,596 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:49,596 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:49,596 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:49,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53558 deadline: 1732648549592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:49,596 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:49,596 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:49,596 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:49,596 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:49,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53568 deadline: 1732648549592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:49,596 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:49,596 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:49,597 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:49,597 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:49,597 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:49,716 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-26T19:14:49,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. 2024-11-26T19:14:49,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. as already flushing 2024-11-26T19:14:49,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. 2024-11-26T19:14:49,717 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:49,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:49,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:49,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-26T19:14:49,806 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:49,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53632 deadline: 1732648549802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:49,807 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:49,807 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:49,807 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:49,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53556 deadline: 1732648549802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:49,807 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:49,807 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:49,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53558 deadline: 1732648549802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:49,807 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:49,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53568 deadline: 1732648549802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:49,808 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:49,808 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:49,808 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:49,808 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:49,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53582 deadline: 1732648549802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:49,808 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:49,808 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:49,808 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:49,808 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:49,808 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:49,808 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:49,808 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:49,808 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:49,808 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:49,859 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/A/80ca1aabf1a841c9aadcdd990dd4c53f 2024-11-26T19:14:49,869 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-26T19:14:49,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. 2024-11-26T19:14:49,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. as already flushing 2024-11-26T19:14:49,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. 2024-11-26T19:14:49,870 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:49,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:49,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:49,895 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/B/99554e299218496cb82c5fa1a4ef37d7 is 50, key is test_row_0/B:col10/1732648489407/Put/seqid=0 2024-11-26T19:14:49,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742091_1267 (size=12001) 2024-11-26T19:14:50,023 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-26T19:14:50,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. 2024-11-26T19:14:50,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. as already flushing 2024-11-26T19:14:50,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. 2024-11-26T19:14:50,023 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:50,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:50,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:50,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-26T19:14:50,115 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:50,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53632 deadline: 1732648550112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:50,116 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:50,116 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:50,116 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:50,116 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:50,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53556 deadline: 1732648550112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:50,116 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:50,116 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:50,116 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:50,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53558 deadline: 1732648550112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:50,116 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:50,117 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:50,117 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:50,117 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:50,117 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:50,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53582 deadline: 1732648550112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:50,117 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:50,117 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:50,117 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:50,124 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:50,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53568 deadline: 1732648550121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:50,124 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:50,124 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:50,124 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:50,176 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-26T19:14:50,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. 2024-11-26T19:14:50,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. as already flushing 2024-11-26T19:14:50,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. 2024-11-26T19:14:50,176 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:50,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:50,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:50,300 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/B/99554e299218496cb82c5fa1a4ef37d7 2024-11-26T19:14:50,327 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/C/33185f0158b44276b42db1e10ea0d23c is 50, key is test_row_0/C:col10/1732648489407/Put/seqid=0 2024-11-26T19:14:50,329 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-26T19:14:50,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. 2024-11-26T19:14:50,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. as already flushing 2024-11-26T19:14:50,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. 2024-11-26T19:14:50,329 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:50,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:50,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:50,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742092_1268 (size=12001) 2024-11-26T19:14:50,482 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-26T19:14:50,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. 2024-11-26T19:14:50,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. as already flushing 2024-11-26T19:14:50,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. 2024-11-26T19:14:50,483 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:50,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:50,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:50,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-26T19:14:50,624 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:50,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53632 deadline: 1732648550621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:50,624 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:50,624 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:50,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53556 deadline: 1732648550622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:50,624 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:50,624 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:50,625 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:50,625 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:50,625 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:50,625 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:50,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53558 deadline: 1732648550622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:50,625 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:50,625 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:50,625 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:50,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53582 deadline: 1732648550622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:50,625 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:50,625 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:50,625 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:50,625 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:50,635 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:50,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53568 deadline: 1732648550632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:50,636 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-26T19:14:50,636 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:50,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. 2024-11-26T19:14:50,636 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:50,636 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:50,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. as already flushing 2024-11-26T19:14:50,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. 2024-11-26T19:14:50,636 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:50,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:50,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:50,732 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/C/33185f0158b44276b42db1e10ea0d23c 2024-11-26T19:14:50,737 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/A/80ca1aabf1a841c9aadcdd990dd4c53f as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/A/80ca1aabf1a841c9aadcdd990dd4c53f 2024-11-26T19:14:50,741 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/A/80ca1aabf1a841c9aadcdd990dd4c53f, entries=200, sequenceid=12, filesize=14.0 K 2024-11-26T19:14:50,742 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/B/99554e299218496cb82c5fa1a4ef37d7 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/B/99554e299218496cb82c5fa1a4ef37d7 2024-11-26T19:14:50,746 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/B/99554e299218496cb82c5fa1a4ef37d7, entries=150, sequenceid=12, filesize=11.7 K 2024-11-26T19:14:50,746 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/C/33185f0158b44276b42db1e10ea0d23c as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/C/33185f0158b44276b42db1e10ea0d23c 2024-11-26T19:14:50,750 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/C/33185f0158b44276b42db1e10ea0d23c, entries=150, sequenceid=12, filesize=11.7 K 2024-11-26T19:14:50,751 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=131.94 KB/135110 for 9fea48dabba44dd37de365711fba859f in 1334ms, sequenceid=12, compaction requested=false 2024-11-26T19:14:50,751 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-26T19:14:50,752 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9fea48dabba44dd37de365711fba859f: 2024-11-26T19:14:50,789 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-26T19:14:50,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. 2024-11-26T19:14:50,789 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2902): Flushing 9fea48dabba44dd37de365711fba859f 3/3 column families, dataSize=131.94 KB heapSize=375.22 KB 2024-11-26T19:14:50,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9fea48dabba44dd37de365711fba859f, store=A 2024-11-26T19:14:50,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T19:14:50,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9fea48dabba44dd37de365711fba859f, store=B 2024-11-26T19:14:50,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T19:14:50,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9fea48dabba44dd37de365711fba859f, store=C 2024-11-26T19:14:50,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T19:14:50,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/A/833460c1b0214807a39188979120517b is 50, key is test_row_0/A:col10/1732648489472/Put/seqid=0 2024-11-26T19:14:50,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742093_1269 (size=12001) 2024-11-26T19:14:50,810 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-26T19:14:51,207 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/A/833460c1b0214807a39188979120517b 2024-11-26T19:14:51,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/B/4da3c279be304242bd6b601b9f924f60 is 50, key is test_row_0/B:col10/1732648489472/Put/seqid=0 2024-11-26T19:14:51,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742094_1270 (size=12001) 2024-11-26T19:14:51,235 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/B/4da3c279be304242bd6b601b9f924f60 2024-11-26T19:14:51,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/C/346cdda36bf5484ba57db7c586fba603 is 50, key is test_row_0/C:col10/1732648489472/Put/seqid=0 2024-11-26T19:14:51,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742095_1271 (size=12001) 2024-11-26T19:14:51,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-26T19:14:51,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(8855): Flush requested on 9fea48dabba44dd37de365711fba859f 2024-11-26T19:14:51,635 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. as already flushing 2024-11-26T19:14:51,652 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/C/346cdda36bf5484ba57db7c586fba603 2024-11-26T19:14:51,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/A/833460c1b0214807a39188979120517b as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/A/833460c1b0214807a39188979120517b 2024-11-26T19:14:51,660 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/A/833460c1b0214807a39188979120517b, entries=150, sequenceid=48, filesize=11.7 K 2024-11-26T19:14:51,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/B/4da3c279be304242bd6b601b9f924f60 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/B/4da3c279be304242bd6b601b9f924f60 2024-11-26T19:14:51,664 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/B/4da3c279be304242bd6b601b9f924f60, entries=150, sequenceid=48, filesize=11.7 K 2024-11-26T19:14:51,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/C/346cdda36bf5484ba57db7c586fba603 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/C/346cdda36bf5484ba57db7c586fba603 2024-11-26T19:14:51,668 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/C/346cdda36bf5484ba57db7c586fba603, entries=150, sequenceid=48, filesize=11.7 K 2024-11-26T19:14:51,679 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(3140): Finished flush of dataSize ~131.94 KB/135110, heapSize ~375.17 KB/384176, currentSize=6.71 KB/6870 for 9fea48dabba44dd37de365711fba859f in 890ms, sequenceid=48, compaction requested=false 2024-11-26T19:14:51,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2603): Flush status journal for 9fea48dabba44dd37de365711fba859f: 2024-11-26T19:14:51,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. 2024-11-26T19:14:51,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=61 2024-11-26T19:14:51,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.HMaster(4169): Remote procedure done, pid=61 2024-11-26T19:14:51,683 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-26T19:14:51,683 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2710 sec 2024-11-26T19:14:51,685 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees in 2.2850 sec 2024-11-26T19:14:51,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(8855): Flush requested on 9fea48dabba44dd37de365711fba859f 2024-11-26T19:14:51,688 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9fea48dabba44dd37de365711fba859f 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-26T19:14:51,689 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9fea48dabba44dd37de365711fba859f, store=A 2024-11-26T19:14:51,689 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:14:51,689 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9fea48dabba44dd37de365711fba859f, store=B 2024-11-26T19:14:51,689 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:14:51,689 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9fea48dabba44dd37de365711fba859f, store=C 2024-11-26T19:14:51,689 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:14:51,694 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/A/25eaa41f1344439cbb967f83f03479a3 is 50, key is test_row_0/A:col10/1732648491685/Put/seqid=0 2024-11-26T19:14:51,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742096_1272 (size=14341) 2024-11-26T19:14:51,719 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732648491707, max timestamp=1732648491715Num uniques 859; 2024-11-26T19:14:51,719 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-26T19:14:51,719 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732648491707, max timestamp=1732648491715Num uniques 859; 2024-11-26T19:14:51,719 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732648491707, max timestamp=1732648491715Num uniques 859; 2024-11-26T19:14:51,719 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-26T19:14:51,719 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-26T19:14:51,745 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:51,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53632 deadline: 1732648551737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:51,746 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:51,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53556 deadline: 1732648551739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:51,746 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:51,746 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:51,746 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:51,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53582 deadline: 1732648551738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:51,746 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:51,746 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:51,746 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:51,746 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:51,746 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:51,747 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:51,747 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:51,748 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:51,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53558 deadline: 1732648551742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:51,748 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:51,748 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:51,749 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:51,752 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:51,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53568 deadline: 1732648551746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:51,752 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:51,752 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:51,752 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:51,855 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:51,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53632 deadline: 1732648551852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:51,856 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:51,856 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:51,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53556 deadline: 1732648551852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:51,856 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:51,856 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:51,856 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:51,856 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:51,856 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:51,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53582 deadline: 1732648551852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:51,856 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:51,856 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:51,856 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:51,856 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:51,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53558 deadline: 1732648551852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:51,856 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:51,857 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:51,857 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:51,857 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:51,863 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:51,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53568 deadline: 1732648551861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:51,864 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:51,864 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:51,864 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:52,066 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:52,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53632 deadline: 1732648552062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:52,066 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:52,066 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:52,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53556 deadline: 1732648552062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:52,067 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:52,067 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:52,067 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:52,067 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:52,067 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:52,067 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:52,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53582 deadline: 1732648552062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:52,068 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:52,068 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:52,068 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:52,068 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:52,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53558 deadline: 1732648552062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:52,068 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:52,068 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:52,068 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:52,076 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:52,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53568 deadline: 1732648552072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:52,076 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:52,076 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:52,076 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:52,100 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/A/25eaa41f1344439cbb967f83f03479a3 2024-11-26T19:14:52,108 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/B/d1c197dd4d7e4364a1852d5387c38afb is 50, key is test_row_0/B:col10/1732648491685/Put/seqid=0 2024-11-26T19:14:52,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742097_1273 (size=12001) 2024-11-26T19:14:52,374 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:52,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53632 deadline: 1732648552372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:52,375 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:52,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53556 deadline: 1732648552372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:52,375 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:52,375 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:52,375 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:52,375 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:52,375 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:52,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53558 deadline: 1732648552372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:52,375 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:52,375 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:52,375 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:52,375 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:52,375 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:52,383 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:52,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53582 deadline: 1732648552381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:52,384 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:52,384 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:52,384 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:52,384 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:52,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53568 deadline: 1732648552382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:52,384 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:52,384 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:52,384 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:52,515 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/B/d1c197dd4d7e4364a1852d5387c38afb 2024-11-26T19:14:52,523 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/C/39c215788bc843a89c221b150d0b3fe6 is 50, key is test_row_0/C:col10/1732648491685/Put/seqid=0 2024-11-26T19:14:52,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742098_1274 (size=12001) 2024-11-26T19:14:52,887 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:52,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53632 deadline: 1732648552882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:52,888 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:52,888 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:52,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53556 deadline: 1732648552882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:52,888 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:52,888 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:52,888 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:52,888 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:52,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53558 deadline: 1732648552882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:52,888 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:52,888 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:52,888 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:52,888 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:52,888 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:52,896 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:52,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53582 deadline: 1732648552891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:52,896 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:52,896 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:52,896 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:52,896 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:52,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53568 deadline: 1732648552891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:52,897 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:52,897 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:52,897 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:52,927 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/C/39c215788bc843a89c221b150d0b3fe6 2024-11-26T19:14:52,932 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/A/25eaa41f1344439cbb967f83f03479a3 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/A/25eaa41f1344439cbb967f83f03479a3 2024-11-26T19:14:52,935 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/A/25eaa41f1344439cbb967f83f03479a3, entries=200, sequenceid=59, filesize=14.0 K 2024-11-26T19:14:52,936 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/B/d1c197dd4d7e4364a1852d5387c38afb as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/B/d1c197dd4d7e4364a1852d5387c38afb 2024-11-26T19:14:52,940 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/B/d1c197dd4d7e4364a1852d5387c38afb, entries=150, sequenceid=59, filesize=11.7 K 2024-11-26T19:14:52,941 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/C/39c215788bc843a89c221b150d0b3fe6 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/C/39c215788bc843a89c221b150d0b3fe6 2024-11-26T19:14:52,945 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/C/39c215788bc843a89c221b150d0b3fe6, entries=150, sequenceid=59, filesize=11.7 K 2024-11-26T19:14:52,945 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 9fea48dabba44dd37de365711fba859f in 1257ms, sequenceid=59, compaction requested=true 2024-11-26T19:14:52,946 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9fea48dabba44dd37de365711fba859f: 2024-11-26T19:14:52,946 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9fea48dabba44dd37de365711fba859f:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T19:14:52,946 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:14:52,946 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T19:14:52,946 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9fea48dabba44dd37de365711fba859f:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T19:14:52,946 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:14:52,946 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T19:14:52,946 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9fea48dabba44dd37de365711fba859f:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T19:14:52,946 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T19:14:52,947 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T19:14:52,947 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40683 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T19:14:52,947 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1541): 9fea48dabba44dd37de365711fba859f/A is initiating minor compaction (all files) 2024-11-26T19:14:52,947 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1541): 9fea48dabba44dd37de365711fba859f/B is initiating minor compaction (all files) 2024-11-26T19:14:52,947 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9fea48dabba44dd37de365711fba859f/B in TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. 2024-11-26T19:14:52,947 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9fea48dabba44dd37de365711fba859f/A in TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. 2024-11-26T19:14:52,947 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/A/80ca1aabf1a841c9aadcdd990dd4c53f, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/A/833460c1b0214807a39188979120517b, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/A/25eaa41f1344439cbb967f83f03479a3] into tmpdir=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp, totalSize=39.7 K 2024-11-26T19:14:52,947 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/B/99554e299218496cb82c5fa1a4ef37d7, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/B/4da3c279be304242bd6b601b9f924f60, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/B/d1c197dd4d7e4364a1852d5387c38afb] into tmpdir=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp, totalSize=35.2 K 2024-11-26T19:14:52,947 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting 99554e299218496cb82c5fa1a4ef37d7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732648489407 2024-11-26T19:14:52,947 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting 80ca1aabf1a841c9aadcdd990dd4c53f, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732648489407 2024-11-26T19:14:52,948 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting 4da3c279be304242bd6b601b9f924f60, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1732648489466 2024-11-26T19:14:52,948 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting 833460c1b0214807a39188979120517b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1732648489466 2024-11-26T19:14:52,948 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting d1c197dd4d7e4364a1852d5387c38afb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1732648491633 2024-11-26T19:14:52,948 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting 25eaa41f1344439cbb967f83f03479a3, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1732648491633 2024-11-26T19:14:52,955 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9fea48dabba44dd37de365711fba859f#B#compaction#225 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T19:14:52,956 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/B/614cf91a41c444ee98514d034f217c58 is 50, key is test_row_0/B:col10/1732648491685/Put/seqid=0 2024-11-26T19:14:52,956 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9fea48dabba44dd37de365711fba859f#A#compaction#224 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T19:14:52,957 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/A/163d814450b14f6989b82eea513c776c is 50, key is test_row_0/A:col10/1732648491685/Put/seqid=0 2024-11-26T19:14:52,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742099_1275 (size=12104) 2024-11-26T19:14:52,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742100_1276 (size=12104) 2024-11-26T19:14:53,377 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/B/614cf91a41c444ee98514d034f217c58 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/B/614cf91a41c444ee98514d034f217c58 2024-11-26T19:14:53,382 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/A/163d814450b14f6989b82eea513c776c as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/A/163d814450b14f6989b82eea513c776c 2024-11-26T19:14:53,383 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9fea48dabba44dd37de365711fba859f/B of 9fea48dabba44dd37de365711fba859f into 614cf91a41c444ee98514d034f217c58(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T19:14:53,383 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9fea48dabba44dd37de365711fba859f: 2024-11-26T19:14:53,383 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., storeName=9fea48dabba44dd37de365711fba859f/B, priority=13, startTime=1732648492946; duration=0sec 2024-11-26T19:14:53,383 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T19:14:53,383 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9fea48dabba44dd37de365711fba859f:B 2024-11-26T19:14:53,383 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T19:14:53,384 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T19:14:53,385 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1541): 9fea48dabba44dd37de365711fba859f/C is initiating minor compaction (all files) 2024-11-26T19:14:53,385 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9fea48dabba44dd37de365711fba859f/C in TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. 2024-11-26T19:14:53,385 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/C/33185f0158b44276b42db1e10ea0d23c, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/C/346cdda36bf5484ba57db7c586fba603, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/C/39c215788bc843a89c221b150d0b3fe6] into tmpdir=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp, totalSize=35.2 K 2024-11-26T19:14:53,385 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting 33185f0158b44276b42db1e10ea0d23c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732648489407 2024-11-26T19:14:53,385 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting 346cdda36bf5484ba57db7c586fba603, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1732648489466 2024-11-26T19:14:53,386 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting 39c215788bc843a89c221b150d0b3fe6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1732648491633 2024-11-26T19:14:53,387 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9fea48dabba44dd37de365711fba859f/A of 9fea48dabba44dd37de365711fba859f into 163d814450b14f6989b82eea513c776c(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T19:14:53,387 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9fea48dabba44dd37de365711fba859f: 2024-11-26T19:14:53,387 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., storeName=9fea48dabba44dd37de365711fba859f/A, priority=13, startTime=1732648492946; duration=0sec 2024-11-26T19:14:53,388 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:14:53,388 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9fea48dabba44dd37de365711fba859f:A 2024-11-26T19:14:53,392 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9fea48dabba44dd37de365711fba859f#C#compaction#226 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T19:14:53,393 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/C/2f9d41ec937741dfa5d7ae2237b7cfd2 is 50, key is test_row_0/C:col10/1732648491685/Put/seqid=0 2024-11-26T19:14:53,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742101_1277 (size=12104) 2024-11-26T19:14:53,401 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/C/2f9d41ec937741dfa5d7ae2237b7cfd2 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/C/2f9d41ec937741dfa5d7ae2237b7cfd2 2024-11-26T19:14:53,405 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9fea48dabba44dd37de365711fba859f/C of 9fea48dabba44dd37de365711fba859f into 2f9d41ec937741dfa5d7ae2237b7cfd2(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T19:14:53,405 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9fea48dabba44dd37de365711fba859f: 2024-11-26T19:14:53,405 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., storeName=9fea48dabba44dd37de365711fba859f/C, priority=13, startTime=1732648492946; duration=0sec 2024-11-26T19:14:53,405 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:14:53,405 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9fea48dabba44dd37de365711fba859f:C 2024-11-26T19:14:53,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-26T19:14:53,582 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T19:14:53,582 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T19:14:53,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees 2024-11-26T19:14:53,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-26T19:14:53,587 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T19:14:53,588 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T19:14:53,588 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T19:14:53,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-26T19:14:53,740 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-26T19:14:53,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. 2024-11-26T19:14:53,741 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2902): Flushing 9fea48dabba44dd37de365711fba859f 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-11-26T19:14:53,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9fea48dabba44dd37de365711fba859f, store=A 2024-11-26T19:14:53,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T19:14:53,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9fea48dabba44dd37de365711fba859f, store=B 2024-11-26T19:14:53,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T19:14:53,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9fea48dabba44dd37de365711fba859f, store=C 2024-11-26T19:14:53,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T19:14:53,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/A/3c261248e40544d09f49478051ef536a is 50, key is test_row_0/A:col10/1732648491736/Put/seqid=0 2024-11-26T19:14:53,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742102_1278 (size=12001) 2024-11-26T19:14:53,752 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/A/3c261248e40544d09f49478051ef536a 2024-11-26T19:14:53,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/B/6ad1a36bf405489badf67747f44426a5 is 50, key is test_row_0/B:col10/1732648491736/Put/seqid=0 2024-11-26T19:14:53,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742103_1279 (size=12001) 2024-11-26T19:14:53,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(8855): Flush requested on 9fea48dabba44dd37de365711fba859f 2024-11-26T19:14:53,895 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. as already flushing 2024-11-26T19:14:53,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-26T19:14:53,914 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:53,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53568 deadline: 1732648553908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:53,914 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:53,914 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:53,915 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:53,915 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:53,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53582 deadline: 1732648553911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:53,915 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to fe76acf732b2,42785,1732648423970 for 'test_row_1' in 9fea48dabba44dd37de365711fba859f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2177 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T19:14:53,915 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:53,915 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:53,915 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:53,918 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:53,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53632 deadline: 1732648553913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:53,919 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:53,919 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:53,919 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:53,921 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:53,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53556 deadline: 1732648553914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:53,921 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:53,921 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:53,921 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:53,921 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:53,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53558 deadline: 1732648553914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:53,922 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:53,922 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:53,922 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:54,023 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:54,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53568 deadline: 1732648554021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:54,024 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:54,024 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:54,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53632 deadline: 1732648554021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:54,024 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:54,024 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:54,024 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:54,024 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:54,024 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:54,035 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:54,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53556 deadline: 1732648554032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:54,035 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:54,036 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:54,036 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:54,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53558 deadline: 1732648554032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:54,036 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:54,036 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:54,036 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:54,036 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:54,180 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/B/6ad1a36bf405489badf67747f44426a5 2024-11-26T19:14:54,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/C/99c6e833533b4d67a1917fc913e5c311 is 50, key is test_row_0/C:col10/1732648491736/Put/seqid=0 2024-11-26T19:14:54,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742104_1280 (size=12001) 2024-11-26T19:14:54,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-26T19:14:54,235 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:54,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53568 deadline: 1732648554231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:54,235 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:54,235 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:54,235 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:54,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53632 deadline: 1732648554232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:54,235 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:54,236 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:54,236 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:54,236 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:54,245 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:54,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53556 deadline: 1732648554242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:54,245 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:54,245 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:54,245 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:54,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53558 deadline: 1732648554242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:54,245 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:54,246 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:54,246 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:54,246 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:54,545 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:54,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53568 deadline: 1732648554542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:54,545 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:54,545 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:54,545 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:54,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53632 deadline: 1732648554542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:54,545 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:54,546 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:54,546 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:54,546 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:54,554 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:54,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53556 deadline: 1732648554552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:54,554 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:54,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53558 deadline: 1732648554552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:54,555 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:54,555 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:54,555 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:54,555 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:54,555 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:54,555 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:54,594 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/C/99c6e833533b4d67a1917fc913e5c311 2024-11-26T19:14:54,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/A/3c261248e40544d09f49478051ef536a as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/A/3c261248e40544d09f49478051ef536a 2024-11-26T19:14:54,603 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/A/3c261248e40544d09f49478051ef536a, entries=150, sequenceid=97, filesize=11.7 K 2024-11-26T19:14:54,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/B/6ad1a36bf405489badf67747f44426a5 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/B/6ad1a36bf405489badf67747f44426a5 2024-11-26T19:14:54,607 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/B/6ad1a36bf405489badf67747f44426a5, entries=150, sequenceid=97, filesize=11.7 K 2024-11-26T19:14:54,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/C/99c6e833533b4d67a1917fc913e5c311 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/C/99c6e833533b4d67a1917fc913e5c311 2024-11-26T19:14:54,614 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/C/99c6e833533b4d67a1917fc913e5c311, entries=150, sequenceid=97, filesize=11.7 K 2024-11-26T19:14:54,615 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=60.38 KB/61830 for 9fea48dabba44dd37de365711fba859f in 875ms, sequenceid=97, compaction requested=false 2024-11-26T19:14:54,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2603): Flush status journal for 9fea48dabba44dd37de365711fba859f: 2024-11-26T19:14:54,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. 2024-11-26T19:14:54,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=63 2024-11-26T19:14:54,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.HMaster(4169): Remote procedure done, pid=63 2024-11-26T19:14:54,618 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-26T19:14:54,618 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0280 sec 2024-11-26T19:14:54,620 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees in 1.0360 sec 2024-11-26T19:14:54,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-26T19:14:54,722 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T19:14:54,722 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T19:14:54,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees 2024-11-26T19:14:54,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-26T19:14:54,724 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T19:14:54,724 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T19:14:54,725 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T19:14:54,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-26T19:14:54,877 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-26T19:14:54,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. 2024-11-26T19:14:54,877 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2902): Flushing 9fea48dabba44dd37de365711fba859f 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-26T19:14:54,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9fea48dabba44dd37de365711fba859f, store=A 2024-11-26T19:14:54,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:14:54,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9fea48dabba44dd37de365711fba859f, store=B 2024-11-26T19:14:54,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:14:54,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9fea48dabba44dd37de365711fba859f, store=C 2024-11-26T19:14:54,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:14:54,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/A/93a725690e5b40fea64ccb781269788f is 50, key is test_row_0/A:col10/1732648493903/Put/seqid=0 2024-11-26T19:14:54,887 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742105_1281 (size=12001) 2024-11-26T19:14:55,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-26T19:14:55,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(8855): Flush requested on 9fea48dabba44dd37de365711fba859f 2024-11-26T19:14:55,054 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. as already flushing 2024-11-26T19:14:55,083 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:55,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53632 deadline: 1732648555078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:55,083 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:55,083 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:55,083 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:55,085 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:55,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53556 deadline: 1732648555080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:55,085 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:55,085 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:55,085 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:55,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53568 deadline: 1732648555081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:55,085 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:55,086 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:55,086 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:55,086 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:55,086 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:55,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53558 deadline: 1732648555082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:55,086 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:55,086 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:55,086 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:55,195 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:55,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53556 deadline: 1732648555192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:55,195 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:55,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53632 deadline: 1732648555192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:55,195 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:55,195 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:55,196 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:55,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53568 deadline: 1732648555192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:55,196 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:55,196 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:55,196 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:55,196 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:55,196 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:55,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53558 deadline: 1732648555192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:55,196 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:55,196 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:55,196 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:55,196 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:55,196 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:55,196 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:55,288 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=109 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/A/93a725690e5b40fea64ccb781269788f 2024-11-26T19:14:55,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/B/ac62361e7ed844ddb5b5b3fd013be089 is 50, key is test_row_0/B:col10/1732648493903/Put/seqid=0 2024-11-26T19:14:55,299 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742106_1282 (size=12001) 2024-11-26T19:14:55,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-26T19:14:55,403 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:55,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53556 deadline: 1732648555402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:55,403 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:55,403 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:55,403 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:55,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53632 deadline: 1732648555402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:55,403 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:55,404 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:55,404 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:55,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53568 deadline: 1732648555402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:55,404 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:55,404 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:55,404 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:55,404 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:55,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53558 deadline: 1732648555402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:55,404 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:55,404 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:55,404 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:55,404 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:55,404 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:55,700 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=109 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/B/ac62361e7ed844ddb5b5b3fd013be089 2024-11-26T19:14:55,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/C/830fa129d23b4d4c8e8726903e499285 is 50, key is test_row_0/C:col10/1732648493903/Put/seqid=0 2024-11-26T19:14:55,714 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:55,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53632 deadline: 1732648555712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:55,714 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:55,714 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:55,715 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:55,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53568 deadline: 1732648555712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:55,715 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:55,715 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:55,715 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:55,715 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:55,715 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:55,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53558 deadline: 1732648555712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:55,715 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:55,715 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:55,715 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:55,715 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:55,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53556 deadline: 1732648555712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:55,716 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:55,716 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:55,716 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:55,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742107_1283 (size=12001) 2024-11-26T19:14:55,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-26T19:14:55,924 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:55,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53582 deadline: 1732648555922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:55,925 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to fe76acf732b2,42785,1732648423970 for 'test_row_1' in 9fea48dabba44dd37de365711fba859f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4186 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T19:14:55,925 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:55,925 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:55,925 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:56,126 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=109 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/C/830fa129d23b4d4c8e8726903e499285 2024-11-26T19:14:56,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/A/93a725690e5b40fea64ccb781269788f as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/A/93a725690e5b40fea64ccb781269788f 2024-11-26T19:14:56,135 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/A/93a725690e5b40fea64ccb781269788f, entries=150, sequenceid=109, filesize=11.7 K 2024-11-26T19:14:56,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/B/ac62361e7ed844ddb5b5b3fd013be089 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/B/ac62361e7ed844ddb5b5b3fd013be089 2024-11-26T19:14:56,141 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/B/ac62361e7ed844ddb5b5b3fd013be089, entries=150, sequenceid=109, filesize=11.7 K 2024-11-26T19:14:56,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/C/830fa129d23b4d4c8e8726903e499285 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/C/830fa129d23b4d4c8e8726903e499285 2024-11-26T19:14:56,147 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/C/830fa129d23b4d4c8e8726903e499285, entries=150, sequenceid=109, filesize=11.7 K 2024-11-26T19:14:56,149 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 9fea48dabba44dd37de365711fba859f in 1271ms, sequenceid=109, compaction requested=true 2024-11-26T19:14:56,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2603): Flush status journal for 9fea48dabba44dd37de365711fba859f: 2024-11-26T19:14:56,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. 2024-11-26T19:14:56,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=65 2024-11-26T19:14:56,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.HMaster(4169): Remote procedure done, pid=65 2024-11-26T19:14:56,156 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-11-26T19:14:56,156 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4280 sec 2024-11-26T19:14:56,158 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees in 1.4350 sec 2024-11-26T19:14:56,225 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9fea48dabba44dd37de365711fba859f 3/3 column families, dataSize=127.47 KB heapSize=379.97 KB 2024-11-26T19:14:56,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(8855): Flush requested on 9fea48dabba44dd37de365711fba859f 2024-11-26T19:14:56,226 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732648495070, max timestamp=1732648496223Num uniques 859; 2024-11-26T19:14:56,226 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-26T19:14:56,226 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732648495070, max timestamp=1732648496223Num uniques 859; 2024-11-26T19:14:56,227 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-26T19:14:56,227 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732648495070, max timestamp=1732648496223Num uniques 859; 2024-11-26T19:14:56,227 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-26T19:14:56,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9fea48dabba44dd37de365711fba859f, store=A 2024-11-26T19:14:56,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T19:14:56,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9fea48dabba44dd37de365711fba859f, store=B 2024-11-26T19:14:56,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T19:14:56,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9fea48dabba44dd37de365711fba859f, store=C 2024-11-26T19:14:56,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T19:14:56,234 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/A/1be93268fce5493eb086274e136292f2 is 50, key is test_row_0/A:col10/1732648495070/Put/seqid=0 2024-11-26T19:14:56,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742108_1284 (size=14491) 2024-11-26T19:14:56,294 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732648496249, max timestamp=1732648496290Num uniques 859; 2024-11-26T19:14:56,294 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-26T19:14:56,294 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732648496249, max timestamp=1732648496290Num uniques 859; 2024-11-26T19:14:56,295 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-26T19:14:56,295 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732648496249, max timestamp=1732648496290Num uniques 859; 2024-11-26T19:14:56,295 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-26T19:14:56,327 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:56,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53632 deadline: 1732648556321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:56,328 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:56,328 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:56,328 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:56,332 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:56,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53558 deadline: 1732648556322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:56,332 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:56,333 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:56,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53568 deadline: 1732648556324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:56,333 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:56,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53556 deadline: 1732648556326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:56,333 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:56,333 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:56,333 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:56,333 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:56,333 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:56,334 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:56,334 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:56,334 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:56,437 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:56,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53632 deadline: 1732648556432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:56,437 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:56,437 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:56,437 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:56,446 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:56,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53558 deadline: 1732648556442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:56,447 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:56,447 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:56,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53556 deadline: 1732648556442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:56,447 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:56,447 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:56,447 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:56,447 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:56,447 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:56,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53568 deadline: 1732648556442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:56,447 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:56,448 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:56,448 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:56,448 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:56,647 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:56,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53632 deadline: 1732648556642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:56,647 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:56,647 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:56,647 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:56,655 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:56,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53558 deadline: 1732648556652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:56,655 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:56,655 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:56,655 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:56,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53556 deadline: 1732648556652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:56,655 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:56,656 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:56,656 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:56,656 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:56,656 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/A/1be93268fce5493eb086274e136292f2 2024-11-26T19:14:56,658 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:56,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53568 deadline: 1732648556652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:56,658 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:56,659 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:56,659 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:56,664 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/B/81747701670d4962af9d8e635fd9142a is 50, key is test_row_0/B:col10/1732648495070/Put/seqid=0 2024-11-26T19:14:56,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742109_1285 (size=12101) 2024-11-26T19:14:56,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-26T19:14:56,882 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T19:14:56,882 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T19:14:56,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-11-26T19:14:56,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-26T19:14:56,884 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T19:14:56,884 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T19:14:56,885 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T19:14:56,957 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:56,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53632 deadline: 1732648556952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:56,958 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:56,958 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:56,958 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:56,965 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:56,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53558 deadline: 1732648556962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:56,966 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:56,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53556 deadline: 1732648556962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:56,966 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:56,966 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:56,966 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:56,966 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:56,966 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:56,966 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:56,977 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:56,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53568 deadline: 1732648556971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:56,978 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:56,978 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:56,978 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:56,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-26T19:14:57,037 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-26T19:14:57,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. 2024-11-26T19:14:57,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. as already flushing 2024-11-26T19:14:57,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. 2024-11-26T19:14:57,037 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:57,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:57,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:57,074 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/B/81747701670d4962af9d8e635fd9142a 2024-11-26T19:14:57,084 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/C/c85c2e9abb874214b64322183a5629b7 is 50, key is test_row_0/C:col10/1732648495070/Put/seqid=0 2024-11-26T19:14:57,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742110_1286 (size=12101) 2024-11-26T19:14:57,190 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-26T19:14:57,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. 2024-11-26T19:14:57,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. as already flushing 2024-11-26T19:14:57,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. 2024-11-26T19:14:57,190 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:57,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:57,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:57,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-26T19:14:57,343 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-26T19:14:57,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. 2024-11-26T19:14:57,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. as already flushing 2024-11-26T19:14:57,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. 2024-11-26T19:14:57,343 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:57,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:57,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:57,467 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:57,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53632 deadline: 1732648557462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:57,468 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:57,468 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:57,468 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:57,475 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:57,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53556 deadline: 1732648557471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:57,476 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:57,476 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:57,476 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:57,486 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:57,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53558 deadline: 1732648557482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:57,486 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:57,486 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:57,486 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:57,488 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/C/c85c2e9abb874214b64322183a5629b7 2024-11-26T19:14:57,495 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/A/1be93268fce5493eb086274e136292f2 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/A/1be93268fce5493eb086274e136292f2 2024-11-26T19:14:57,496 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-26T19:14:57,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. 2024-11-26T19:14:57,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. as already flushing 2024-11-26T19:14:57,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. 2024-11-26T19:14:57,497 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:57,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:57,497 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:57,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53568 deadline: 1732648557492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:57,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:57,501 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:57,501 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:57,501 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:57,505 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/A/1be93268fce5493eb086274e136292f2, entries=200, sequenceid=132, filesize=14.2 K 2024-11-26T19:14:57,506 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/B/81747701670d4962af9d8e635fd9142a as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/B/81747701670d4962af9d8e635fd9142a 2024-11-26T19:14:57,510 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/B/81747701670d4962af9d8e635fd9142a, entries=150, sequenceid=132, filesize=11.8 K 2024-11-26T19:14:57,511 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/C/c85c2e9abb874214b64322183a5629b7 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/C/c85c2e9abb874214b64322183a5629b7 2024-11-26T19:14:57,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-26T19:14:57,516 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/C/c85c2e9abb874214b64322183a5629b7, entries=150, sequenceid=132, filesize=11.8 K 2024-11-26T19:14:57,517 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.03 KB/108576, currentSize=147.60 KB/151140 for 9fea48dabba44dd37de365711fba859f in 1292ms, sequenceid=132, compaction requested=true 2024-11-26T19:14:57,517 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9fea48dabba44dd37de365711fba859f: 2024-11-26T19:14:57,517 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9fea48dabba44dd37de365711fba859f:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T19:14:57,517 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:14:57,517 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T19:14:57,517 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9fea48dabba44dd37de365711fba859f:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T19:14:57,517 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:14:57,517 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9fea48dabba44dd37de365711fba859f:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T19:14:57,517 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T19:14:57,517 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T19:14:57,518 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48207 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T19:14:57,518 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50597 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T19:14:57,519 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1541): 9fea48dabba44dd37de365711fba859f/A is initiating minor compaction (all files) 2024-11-26T19:14:57,519 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9fea48dabba44dd37de365711fba859f/A in TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. 2024-11-26T19:14:57,519 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/A/163d814450b14f6989b82eea513c776c, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/A/3c261248e40544d09f49478051ef536a, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/A/93a725690e5b40fea64ccb781269788f, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/A/1be93268fce5493eb086274e136292f2] into tmpdir=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp, totalSize=49.4 K 2024-11-26T19:14:57,519 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1541): 9fea48dabba44dd37de365711fba859f/B is initiating minor compaction (all files) 2024-11-26T19:14:57,519 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9fea48dabba44dd37de365711fba859f/B in TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. 2024-11-26T19:14:57,519 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/B/614cf91a41c444ee98514d034f217c58, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/B/6ad1a36bf405489badf67747f44426a5, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/B/ac62361e7ed844ddb5b5b3fd013be089, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/B/81747701670d4962af9d8e635fd9142a] into tmpdir=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp, totalSize=47.1 K 2024-11-26T19:14:57,519 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting 163d814450b14f6989b82eea513c776c, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1732648491633 2024-11-26T19:14:57,520 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting 614cf91a41c444ee98514d034f217c58, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1732648491633 2024-11-26T19:14:57,520 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3c261248e40544d09f49478051ef536a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1732648491725 2024-11-26T19:14:57,520 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting 6ad1a36bf405489badf67747f44426a5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1732648491725 2024-11-26T19:14:57,520 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting 93a725690e5b40fea64ccb781269788f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1732648493903 2024-11-26T19:14:57,521 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1be93268fce5493eb086274e136292f2, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732648495070 2024-11-26T19:14:57,521 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting ac62361e7ed844ddb5b5b3fd013be089, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1732648493903 2024-11-26T19:14:57,521 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting 81747701670d4962af9d8e635fd9142a, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732648495070 2024-11-26T19:14:57,535 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9fea48dabba44dd37de365711fba859f#B#compaction#236 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T19:14:57,535 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9fea48dabba44dd37de365711fba859f#A#compaction#237 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T19:14:57,536 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/B/6dce579f4c34441bbad425f55936e1a4 is 50, key is test_row_0/B:col10/1732648495070/Put/seqid=0 2024-11-26T19:14:57,536 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/A/7323d6afe09546deb22c22e22bd77e61 is 50, key is test_row_0/A:col10/1732648495070/Put/seqid=0 2024-11-26T19:14:57,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742111_1287 (size=12341) 2024-11-26T19:14:57,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742112_1288 (size=12341) 2024-11-26T19:14:57,652 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-26T19:14:57,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. 2024-11-26T19:14:57,653 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing 9fea48dabba44dd37de365711fba859f 3/3 column families, dataSize=147.60 KB heapSize=415.45 KB 2024-11-26T19:14:57,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9fea48dabba44dd37de365711fba859f, store=A 2024-11-26T19:14:57,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T19:14:57,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9fea48dabba44dd37de365711fba859f, store=B 2024-11-26T19:14:57,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T19:14:57,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9fea48dabba44dd37de365711fba859f, store=C 2024-11-26T19:14:57,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T19:14:57,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/A/057584545c75402c8894f48bb1c4f5ee is 50, key is test_row_0/A:col10/1732648496322/Put/seqid=0 2024-11-26T19:14:57,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742113_1289 (size=12151) 2024-11-26T19:14:57,945 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/B/6dce579f4c34441bbad425f55936e1a4 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/B/6dce579f4c34441bbad425f55936e1a4 2024-11-26T19:14:57,950 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9fea48dabba44dd37de365711fba859f/B of 9fea48dabba44dd37de365711fba859f into 6dce579f4c34441bbad425f55936e1a4(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T19:14:57,950 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9fea48dabba44dd37de365711fba859f: 2024-11-26T19:14:57,950 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., storeName=9fea48dabba44dd37de365711fba859f/B, priority=12, startTime=1732648497517; duration=0sec 2024-11-26T19:14:57,950 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T19:14:57,950 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9fea48dabba44dd37de365711fba859f:B 2024-11-26T19:14:57,950 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T19:14:57,951 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48207 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T19:14:57,951 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1541): 9fea48dabba44dd37de365711fba859f/C is initiating minor compaction (all files) 2024-11-26T19:14:57,952 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9fea48dabba44dd37de365711fba859f/C in TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. 2024-11-26T19:14:57,952 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/C/2f9d41ec937741dfa5d7ae2237b7cfd2, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/C/99c6e833533b4d67a1917fc913e5c311, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/C/830fa129d23b4d4c8e8726903e499285, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/C/c85c2e9abb874214b64322183a5629b7] into tmpdir=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp, totalSize=47.1 K 2024-11-26T19:14:57,952 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting 2f9d41ec937741dfa5d7ae2237b7cfd2, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1732648491633 2024-11-26T19:14:57,952 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting 99c6e833533b4d67a1917fc913e5c311, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1732648491725 2024-11-26T19:14:57,953 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting 830fa129d23b4d4c8e8726903e499285, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1732648493903 2024-11-26T19:14:57,953 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting c85c2e9abb874214b64322183a5629b7, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732648495070 2024-11-26T19:14:57,961 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9fea48dabba44dd37de365711fba859f#C#compaction#239 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T19:14:57,962 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/C/e219d6fa7b454ece86951192720086e3 is 50, key is test_row_0/C:col10/1732648495070/Put/seqid=0 2024-11-26T19:14:57,969 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/A/7323d6afe09546deb22c22e22bd77e61 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/A/7323d6afe09546deb22c22e22bd77e61 2024-11-26T19:14:57,975 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9fea48dabba44dd37de365711fba859f/A of 9fea48dabba44dd37de365711fba859f into 7323d6afe09546deb22c22e22bd77e61(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T19:14:57,975 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9fea48dabba44dd37de365711fba859f: 2024-11-26T19:14:57,975 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., storeName=9fea48dabba44dd37de365711fba859f/A, priority=12, startTime=1732648497517; duration=0sec 2024-11-26T19:14:57,975 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:14:57,975 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9fea48dabba44dd37de365711fba859f:A 2024-11-26T19:14:57,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742114_1290 (size=12341) 2024-11-26T19:14:57,995 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/C/e219d6fa7b454ece86951192720086e3 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/C/e219d6fa7b454ece86951192720086e3 2024-11-26T19:14:58,001 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9fea48dabba44dd37de365711fba859f/C of 9fea48dabba44dd37de365711fba859f into e219d6fa7b454ece86951192720086e3(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T19:14:58,001 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9fea48dabba44dd37de365711fba859f: 2024-11-26T19:14:58,001 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., storeName=9fea48dabba44dd37de365711fba859f/C, priority=12, startTime=1732648497517; duration=0sec 2024-11-26T19:14:58,001 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:14:58,001 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9fea48dabba44dd37de365711fba859f:C 2024-11-26T19:14:58,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-26T19:14:58,062 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/A/057584545c75402c8894f48bb1c4f5ee 2024-11-26T19:14:58,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/B/f4779b77e81d4463904ba48cf953b2d1 is 50, key is test_row_0/B:col10/1732648496322/Put/seqid=0 2024-11-26T19:14:58,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742115_1291 (size=12151) 2024-11-26T19:14:58,478 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/B/f4779b77e81d4463904ba48cf953b2d1 2024-11-26T19:14:58,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(8855): Flush requested on 9fea48dabba44dd37de365711fba859f 2024-11-26T19:14:58,484 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. as already flushing 2024-11-26T19:14:58,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/C/7d918e0ee7324aaf9702f798c320d8e2 is 50, key is test_row_0/C:col10/1732648496322/Put/seqid=0 2024-11-26T19:14:58,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742116_1292 (size=12151) 2024-11-26T19:14:58,496 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:58,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53556 deadline: 1732648558492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:58,496 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:58,496 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:58,496 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:58,501 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:58,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53632 deadline: 1732648558496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:58,502 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:58,502 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:58,502 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:58,502 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:58,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53558 deadline: 1732648558496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:58,502 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:58,502 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:58,502 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:58,515 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:58,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53568 deadline: 1732648558512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:58,515 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to fe76acf732b2,42785,1732648423970 for 'test_row_2' in 9fea48dabba44dd37de365711fba859f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2191 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T19:14:58,516 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:58,516 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:58,516 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:58,605 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:58,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53556 deadline: 1732648558602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:58,606 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:58,606 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:58,606 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:58,616 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:58,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53558 deadline: 1732648558612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:58,616 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:58,616 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:58,616 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:58,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53632 deadline: 1732648558612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:58,616 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:58,617 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:58,617 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:58,617 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:58,814 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:58,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53556 deadline: 1732648558812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:58,814 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:58,814 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:58,814 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:58,826 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:58,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53558 deadline: 1732648558822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:58,826 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:58,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53632 deadline: 1732648558822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:58,826 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:58,826 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:58,826 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:58,826 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:58,826 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:58,826 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:58,892 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/C/7d918e0ee7324aaf9702f798c320d8e2 2024-11-26T19:14:58,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/A/057584545c75402c8894f48bb1c4f5ee as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/A/057584545c75402c8894f48bb1c4f5ee 2024-11-26T19:14:58,902 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/A/057584545c75402c8894f48bb1c4f5ee, entries=150, sequenceid=169, filesize=11.9 K 2024-11-26T19:14:58,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/B/f4779b77e81d4463904ba48cf953b2d1 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/B/f4779b77e81d4463904ba48cf953b2d1 2024-11-26T19:14:58,906 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/B/f4779b77e81d4463904ba48cf953b2d1, entries=150, sequenceid=169, filesize=11.9 K 2024-11-26T19:14:58,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/C/7d918e0ee7324aaf9702f798c320d8e2 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/C/7d918e0ee7324aaf9702f798c320d8e2 2024-11-26T19:14:58,911 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/C/7d918e0ee7324aaf9702f798c320d8e2, entries=150, sequenceid=169, filesize=11.9 K 2024-11-26T19:14:58,911 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~415.41 KB/425376, currentSize=40.25 KB/41220 for 9fea48dabba44dd37de365711fba859f in 1258ms, sequenceid=169, compaction requested=false 2024-11-26T19:14:58,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for 9fea48dabba44dd37de365711fba859f: 2024-11-26T19:14:58,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. 2024-11-26T19:14:58,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-11-26T19:14:58,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-11-26T19:14:58,915 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-11-26T19:14:58,915 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0290 sec 2024-11-26T19:14:58,918 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 2.0340 sec 2024-11-26T19:14:59,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-26T19:14:59,032 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T19:14:59,033 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T19:14:59,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-11-26T19:14:59,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-26T19:14:59,035 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T19:14:59,036 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T19:14:59,036 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T19:14:59,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(8855): Flush requested on 9fea48dabba44dd37de365711fba859f 2024-11-26T19:14:59,133 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9fea48dabba44dd37de365711fba859f 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-26T19:14:59,133 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9fea48dabba44dd37de365711fba859f, store=A 2024-11-26T19:14:59,134 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:14:59,134 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9fea48dabba44dd37de365711fba859f, store=B 2024-11-26T19:14:59,134 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:14:59,134 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9fea48dabba44dd37de365711fba859f, store=C 2024-11-26T19:14:59,134 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:14:59,139 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/A/94c7589f7a504502aa7b48d516454d01 is 50, key is test_row_0/A:col10/1732648498495/Put/seqid=0 2024-11-26T19:14:59,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-26T19:14:59,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742117_1293 (size=14541) 2024-11-26T19:14:59,146 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=182 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/A/94c7589f7a504502aa7b48d516454d01 2024-11-26T19:14:59,160 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/B/a49a748b1b4c499397e7207d90c7a0aa is 50, key is test_row_0/B:col10/1732648498495/Put/seqid=0 2024-11-26T19:14:59,182 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732648499161, max timestamp=1732648499177Num uniques 859; 2024-11-26T19:14:59,182 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-26T19:14:59,183 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732648499161, max timestamp=1732648499177Num uniques 859; 2024-11-26T19:14:59,183 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-26T19:14:59,183 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732648499161, max timestamp=1732648499177Num uniques 859; 2024-11-26T19:14:59,184 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-26T19:14:59,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742118_1294 (size=12151) 2024-11-26T19:14:59,185 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=182 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/B/a49a748b1b4c499397e7207d90c7a0aa 2024-11-26T19:14:59,192 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-26T19:14:59,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. 2024-11-26T19:14:59,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. as already flushing 2024-11-26T19:14:59,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. 2024-11-26T19:14:59,192 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:59,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:59,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:59,196 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/C/d3f5af72a5084ea298a468a57385e222 is 50, key is test_row_0/C:col10/1732648498495/Put/seqid=0 2024-11-26T19:14:59,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742119_1295 (size=12151) 2024-11-26T19:14:59,201 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=182 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/C/d3f5af72a5084ea298a468a57385e222 2024-11-26T19:14:59,210 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/A/94c7589f7a504502aa7b48d516454d01 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/A/94c7589f7a504502aa7b48d516454d01 2024-11-26T19:14:59,215 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/A/94c7589f7a504502aa7b48d516454d01, entries=200, sequenceid=182, filesize=14.2 K 2024-11-26T19:14:59,217 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/B/a49a748b1b4c499397e7207d90c7a0aa as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/B/a49a748b1b4c499397e7207d90c7a0aa 2024-11-26T19:14:59,223 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/B/a49a748b1b4c499397e7207d90c7a0aa, entries=150, sequenceid=182, filesize=11.9 K 2024-11-26T19:14:59,224 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/C/d3f5af72a5084ea298a468a57385e222 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/C/d3f5af72a5084ea298a468a57385e222 2024-11-26T19:14:59,229 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/C/d3f5af72a5084ea298a468a57385e222, entries=150, sequenceid=182, filesize=11.9 K 2024-11-26T19:14:59,230 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=127.47 KB/130530 for 9fea48dabba44dd37de365711fba859f in 97ms, sequenceid=182, compaction requested=true 2024-11-26T19:14:59,230 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9fea48dabba44dd37de365711fba859f: 2024-11-26T19:14:59,230 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T19:14:59,231 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9fea48dabba44dd37de365711fba859f:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T19:14:59,231 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:14:59,231 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9fea48dabba44dd37de365711fba859f:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T19:14:59,231 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:14:59,231 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9fea48dabba44dd37de365711fba859f:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T19:14:59,231 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T19:14:59,231 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T19:14:59,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(8855): Flush requested on 9fea48dabba44dd37de365711fba859f 2024-11-26T19:14:59,232 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39033 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T19:14:59,232 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9fea48dabba44dd37de365711fba859f 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-11-26T19:14:59,232 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1541): 9fea48dabba44dd37de365711fba859f/A is initiating minor compaction (all files) 2024-11-26T19:14:59,232 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9fea48dabba44dd37de365711fba859f/A in TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. 2024-11-26T19:14:59,232 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9fea48dabba44dd37de365711fba859f, store=A 2024-11-26T19:14:59,232 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/A/7323d6afe09546deb22c22e22bd77e61, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/A/057584545c75402c8894f48bb1c4f5ee, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/A/94c7589f7a504502aa7b48d516454d01] into tmpdir=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp, totalSize=38.1 K 2024-11-26T19:14:59,232 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T19:14:59,232 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9fea48dabba44dd37de365711fba859f, store=B 2024-11-26T19:14:59,232 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T19:14:59,232 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9fea48dabba44dd37de365711fba859f, store=C 2024-11-26T19:14:59,232 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T19:14:59,232 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36643 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T19:14:59,232 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1541): 9fea48dabba44dd37de365711fba859f/B is initiating minor compaction (all files) 2024-11-26T19:14:59,232 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9fea48dabba44dd37de365711fba859f/B in TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. 2024-11-26T19:14:59,233 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/B/6dce579f4c34441bbad425f55936e1a4, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/B/f4779b77e81d4463904ba48cf953b2d1, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/B/a49a748b1b4c499397e7207d90c7a0aa] into tmpdir=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp, totalSize=35.8 K 2024-11-26T19:14:59,233 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7323d6afe09546deb22c22e22bd77e61, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732648495070 2024-11-26T19:14:59,233 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting 6dce579f4c34441bbad425f55936e1a4, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732648495070 2024-11-26T19:14:59,233 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting 057584545c75402c8894f48bb1c4f5ee, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1732648496319 2024-11-26T19:14:59,234 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting f4779b77e81d4463904ba48cf953b2d1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1732648496319 2024-11-26T19:14:59,234 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting 94c7589f7a504502aa7b48d516454d01, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1732648498490 2024-11-26T19:14:59,234 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting a49a748b1b4c499397e7207d90c7a0aa, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1732648498490 2024-11-26T19:14:59,238 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/A/d684c2e66f464eabb838c5dbadc5bbb1 is 50, key is test_row_0/A:col10/1732648499211/Put/seqid=0 2024-11-26T19:14:59,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742120_1296 (size=16931) 2024-11-26T19:14:59,247 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9fea48dabba44dd37de365711fba859f#A#compaction#246 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T19:14:59,247 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/A/f15b8d4d44c14dd990ef71f87db6bc4c is 50, key is test_row_0/A:col10/1732648498495/Put/seqid=0 2024-11-26T19:14:59,256 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=219 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/A/d684c2e66f464eabb838c5dbadc5bbb1 2024-11-26T19:14:59,260 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:59,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53556 deadline: 1732648559253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:59,260 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:59,260 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:59,260 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:59,260 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:59,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53632 deadline: 1732648559253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:59,261 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:59,261 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:59,261 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:59,267 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9fea48dabba44dd37de365711fba859f#B#compaction#247 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T19:14:59,267 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/B/afa345985f954ec09e440a65816b226d is 50, key is test_row_0/B:col10/1732648498495/Put/seqid=0 2024-11-26T19:14:59,267 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:59,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53558 deadline: 1732648559261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:59,268 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:59,268 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:59,268 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:59,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742121_1297 (size=12493) 2024-11-26T19:14:59,279 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/A/f15b8d4d44c14dd990ef71f87db6bc4c as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/A/f15b8d4d44c14dd990ef71f87db6bc4c 2024-11-26T19:14:59,283 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/B/8742c686f20c472eb1cd0f7b30a9efd6 is 50, key is test_row_0/B:col10/1732648499211/Put/seqid=0 2024-11-26T19:14:59,285 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9fea48dabba44dd37de365711fba859f/A of 9fea48dabba44dd37de365711fba859f into f15b8d4d44c14dd990ef71f87db6bc4c(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T19:14:59,285 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9fea48dabba44dd37de365711fba859f: 2024-11-26T19:14:59,285 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., storeName=9fea48dabba44dd37de365711fba859f/A, priority=13, startTime=1732648499230; duration=0sec 2024-11-26T19:14:59,285 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T19:14:59,285 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9fea48dabba44dd37de365711fba859f:A 2024-11-26T19:14:59,285 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T19:14:59,287 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36643 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T19:14:59,287 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1541): 9fea48dabba44dd37de365711fba859f/C is initiating minor compaction (all files) 2024-11-26T19:14:59,287 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9fea48dabba44dd37de365711fba859f/C in TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. 2024-11-26T19:14:59,287 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/C/e219d6fa7b454ece86951192720086e3, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/C/7d918e0ee7324aaf9702f798c320d8e2, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/C/d3f5af72a5084ea298a468a57385e222] into tmpdir=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp, totalSize=35.8 K 2024-11-26T19:14:59,287 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting e219d6fa7b454ece86951192720086e3, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732648495070 2024-11-26T19:14:59,288 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7d918e0ee7324aaf9702f798c320d8e2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1732648496319 2024-11-26T19:14:59,288 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting d3f5af72a5084ea298a468a57385e222, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1732648498490 2024-11-26T19:14:59,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742122_1298 (size=12493) 2024-11-26T19:14:59,325 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9fea48dabba44dd37de365711fba859f#C#compaction#249 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T19:14:59,326 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/C/8d65a7b76005450db2e45cf5365dcb26 is 50, key is test_row_0/C:col10/1732648498495/Put/seqid=0 2024-11-26T19:14:59,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742123_1299 (size=12151) 2024-11-26T19:14:59,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742124_1300 (size=12493) 2024-11-26T19:14:59,347 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-26T19:14:59,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. 2024-11-26T19:14:59,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. as already flushing 2024-11-26T19:14:59,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. 2024-11-26T19:14:59,347 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:59,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:59,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:59,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-26T19:14:59,359 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/C/8d65a7b76005450db2e45cf5365dcb26 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/C/8d65a7b76005450db2e45cf5365dcb26 2024-11-26T19:14:59,375 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9fea48dabba44dd37de365711fba859f/C of 9fea48dabba44dd37de365711fba859f into 8d65a7b76005450db2e45cf5365dcb26(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T19:14:59,375 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9fea48dabba44dd37de365711fba859f: 2024-11-26T19:14:59,375 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., storeName=9fea48dabba44dd37de365711fba859f/C, priority=13, startTime=1732648499231; duration=0sec 2024-11-26T19:14:59,375 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:14:59,375 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9fea48dabba44dd37de365711fba859f:C 2024-11-26T19:14:59,376 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:59,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53556 deadline: 1732648559372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:59,377 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:59,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53632 deadline: 1732648559372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:59,377 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:59,377 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:59,377 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:59,377 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:59,377 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:59,377 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:59,377 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:59,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53558 deadline: 1732648559372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:59,378 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:59,378 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:59,378 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:59,501 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-26T19:14:59,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. 2024-11-26T19:14:59,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. as already flushing 2024-11-26T19:14:59,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. 2024-11-26T19:14:59,502 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:59,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:59,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:59,586 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:59,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53632 deadline: 1732648559582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:59,587 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:59,587 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:59,587 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:59,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53558 deadline: 1732648559582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:59,587 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:59,587 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:59,587 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:59,587 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:59,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53556 deadline: 1732648559583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:59,587 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:59,588 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:59,588 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:59,588 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:59,655 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-26T19:14:59,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. 2024-11-26T19:14:59,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. as already flushing 2024-11-26T19:14:59,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. 2024-11-26T19:14:59,655 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:59,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:59,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:59,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-26T19:14:59,713 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/B/afa345985f954ec09e440a65816b226d as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/B/afa345985f954ec09e440a65816b226d 2024-11-26T19:14:59,721 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9fea48dabba44dd37de365711fba859f/B of 9fea48dabba44dd37de365711fba859f into afa345985f954ec09e440a65816b226d(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T19:14:59,721 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9fea48dabba44dd37de365711fba859f: 2024-11-26T19:14:59,721 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., storeName=9fea48dabba44dd37de365711fba859f/B, priority=13, startTime=1732648499231; duration=0sec 2024-11-26T19:14:59,721 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:14:59,721 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9fea48dabba44dd37de365711fba859f:B 2024-11-26T19:14:59,731 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=219 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/B/8742c686f20c472eb1cd0f7b30a9efd6 2024-11-26T19:14:59,747 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/C/06dfc70f3cde46b39562ce542b6ad0d1 is 50, key is test_row_0/C:col10/1732648499211/Put/seqid=0 2024-11-26T19:14:59,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742125_1301 (size=12151) 2024-11-26T19:14:59,809 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-26T19:14:59,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. 2024-11-26T19:14:59,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. as already flushing 2024-11-26T19:14:59,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. 2024-11-26T19:14:59,809 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:59,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:59,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:59,896 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:59,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53558 deadline: 1732648559892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:59,897 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:59,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53556 deadline: 1732648559892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:59,897 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:59,897 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:59,897 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:59,897 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:59,897 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:59,897 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:59,895 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:59,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53632 deadline: 1732648559892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:59,899 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:59,899 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:59,899 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:59,944 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:59,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53582 deadline: 1732648559942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:14:59,945 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to fe76acf732b2,42785,1732648423970 for 'test_row_1' in 9fea48dabba44dd37de365711fba859f of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8207 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T19:14:59,945 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:59,945 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:14:59,945 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:14:59,962 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-26T19:14:59,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. 2024-11-26T19:14:59,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. as already flushing 2024-11-26T19:14:59,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. 2024-11-26T19:14:59,962 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:59,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:59,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:00,116 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-26T19:15:00,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. 2024-11-26T19:15:00,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. as already flushing 2024-11-26T19:15:00,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. 2024-11-26T19:15:00,116 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:00,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:00,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:00,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-26T19:15:00,175 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=219 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/C/06dfc70f3cde46b39562ce542b6ad0d1 2024-11-26T19:15:00,180 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/A/d684c2e66f464eabb838c5dbadc5bbb1 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/A/d684c2e66f464eabb838c5dbadc5bbb1 2024-11-26T19:15:00,184 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/A/d684c2e66f464eabb838c5dbadc5bbb1, entries=250, sequenceid=219, filesize=16.5 K 2024-11-26T19:15:00,185 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/B/8742c686f20c472eb1cd0f7b30a9efd6 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/B/8742c686f20c472eb1cd0f7b30a9efd6 2024-11-26T19:15:00,190 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/B/8742c686f20c472eb1cd0f7b30a9efd6, entries=150, sequenceid=219, filesize=11.9 K 2024-11-26T19:15:00,191 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/C/06dfc70f3cde46b39562ce542b6ad0d1 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/C/06dfc70f3cde46b39562ce542b6ad0d1 2024-11-26T19:15:00,195 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/C/06dfc70f3cde46b39562ce542b6ad0d1, entries=150, sequenceid=219, filesize=11.9 K 2024-11-26T19:15:00,196 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=40.25 KB/41220 for 9fea48dabba44dd37de365711fba859f in 964ms, sequenceid=219, compaction requested=false 2024-11-26T19:15:00,196 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9fea48dabba44dd37de365711fba859f: 2024-11-26T19:15:00,269 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-26T19:15:00,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. 2024-11-26T19:15:00,269 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing 9fea48dabba44dd37de365711fba859f 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-26T19:15:00,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9fea48dabba44dd37de365711fba859f, store=A 2024-11-26T19:15:00,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:15:00,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9fea48dabba44dd37de365711fba859f, store=B 2024-11-26T19:15:00,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:15:00,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9fea48dabba44dd37de365711fba859f, store=C 2024-11-26T19:15:00,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:15:00,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/A/ed4375d1e1594ead8299fb77e3ea7be3 is 50, key is test_row_0/A:col10/1732648499258/Put/seqid=0 2024-11-26T19:15:00,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742126_1302 (size=12151) 2024-11-26T19:15:00,321 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/A/ed4375d1e1594ead8299fb77e3ea7be3 2024-11-26T19:15:00,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/B/40c09f8563cf4277a86aa84f990367ae is 50, key is test_row_0/B:col10/1732648499258/Put/seqid=0 2024-11-26T19:15:00,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742127_1303 (size=12151) 2024-11-26T19:15:00,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(8855): Flush requested on 9fea48dabba44dd37de365711fba859f 2024-11-26T19:15:00,404 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. as already flushing 2024-11-26T19:15:00,443 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732648500432, max timestamp=1732648500439Num uniques 859; 2024-11-26T19:15:00,443 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-26T19:15:00,443 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732648500432, max timestamp=1732648500439Num uniques 859; 2024-11-26T19:15:00,443 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732648500432, max timestamp=1732648500439Num uniques 859; 2024-11-26T19:15:00,443 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-26T19:15:00,443 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-26T19:15:00,478 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:00,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53558 deadline: 1732648560473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:00,480 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:00,480 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:00,480 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:00,481 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732648500463, max timestamp=1732648500476Num uniques 1018; 2024-11-26T19:15:00,481 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732648500463, max timestamp=1732648500476Num uniques 1018; 2024-11-26T19:15:00,481 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732648500463, max timestamp=1732648500476Num uniques 1018; 2024-11-26T19:15:00,481 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-26T19:15:00,481 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-26T19:15:00,481 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-26T19:15:00,567 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:00,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53568 deadline: 1732648560562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:00,568 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:00,568 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:00,568 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:00,568 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:00,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53632 deadline: 1732648560563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:00,568 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:00,568 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:00,568 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:00,571 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:00,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53556 deadline: 1732648560565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:00,571 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:00,571 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:00,571 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:00,596 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:00,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53558 deadline: 1732648560591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:00,597 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:00,597 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:00,597 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:00,676 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:00,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53568 deadline: 1732648560672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:00,676 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:00,676 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:00,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53632 deadline: 1732648560672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:00,676 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:00,676 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:00,676 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:00,676 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:00,676 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:00,683 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:00,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53556 deadline: 1732648560681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:00,684 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:00,684 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:00,684 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:00,771 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/B/40c09f8563cf4277a86aa84f990367ae 2024-11-26T19:15:00,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/C/870540b8ca6d4daa94b00196a17f1e7c is 50, key is test_row_0/C:col10/1732648499258/Put/seqid=0 2024-11-26T19:15:00,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742128_1304 (size=12151) 2024-11-26T19:15:00,782 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/C/870540b8ca6d4daa94b00196a17f1e7c 2024-11-26T19:15:00,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/A/ed4375d1e1594ead8299fb77e3ea7be3 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/A/ed4375d1e1594ead8299fb77e3ea7be3 2024-11-26T19:15:00,797 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/A/ed4375d1e1594ead8299fb77e3ea7be3, entries=150, sequenceid=231, filesize=11.9 K 2024-11-26T19:15:00,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/B/40c09f8563cf4277a86aa84f990367ae as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/B/40c09f8563cf4277a86aa84f990367ae 2024-11-26T19:15:00,802 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/B/40c09f8563cf4277a86aa84f990367ae, entries=150, sequenceid=231, filesize=11.9 K 2024-11-26T19:15:00,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/C/870540b8ca6d4daa94b00196a17f1e7c as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/C/870540b8ca6d4daa94b00196a17f1e7c 2024-11-26T19:15:00,803 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:00,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53558 deadline: 1732648560801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:00,804 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:00,804 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:00,804 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:00,806 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/C/870540b8ca6d4daa94b00196a17f1e7c, entries=150, sequenceid=231, filesize=11.9 K 2024-11-26T19:15:00,807 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=140.89 KB/144270 for 9fea48dabba44dd37de365711fba859f in 538ms, sequenceid=231, compaction requested=true 2024-11-26T19:15:00,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for 9fea48dabba44dd37de365711fba859f: 2024-11-26T19:15:00,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. 2024-11-26T19:15:00,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-11-26T19:15:00,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-11-26T19:15:00,810 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-26T19:15:00,811 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7730 sec 2024-11-26T19:15:00,813 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 1.7780 sec 2024-11-26T19:15:00,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(8855): Flush requested on 9fea48dabba44dd37de365711fba859f 2024-11-26T19:15:00,885 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9fea48dabba44dd37de365711fba859f 3/3 column families, dataSize=147.60 KB heapSize=420 KB 2024-11-26T19:15:00,886 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9fea48dabba44dd37de365711fba859f, store=A 2024-11-26T19:15:00,886 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-26T19:15:00,886 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9fea48dabba44dd37de365711fba859f, store=B 2024-11-26T19:15:00,886 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-26T19:15:00,886 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9fea48dabba44dd37de365711fba859f, store=C 2024-11-26T19:15:00,886 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-26T19:15:00,893 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/A/ef8afb7318294433b9f12792fa861007 is 50, key is test_row_0/A:col10/1732648500485/Put/seqid=0 2024-11-26T19:15:00,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742129_1305 (size=14741) 2024-11-26T19:15:00,898 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/A/ef8afb7318294433b9f12792fa861007 2024-11-26T19:15:00,903 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:00,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53632 deadline: 1732648560896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:00,903 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:00,903 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:00,903 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:00,903 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:00,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53556 deadline: 1732648560896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:00,904 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:00,904 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:00,904 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:00,906 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:00,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53568 deadline: 1732648560901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:00,906 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:00,906 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:00,906 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:00,911 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/B/0f1594892dbe429b947e32e365b37685 is 50, key is test_row_0/B:col10/1732648500485/Put/seqid=0 2024-11-26T19:15:00,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742130_1306 (size=12301) 2024-11-26T19:15:01,016 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:01,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53632 deadline: 1732648561011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:01,016 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:01,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53556 deadline: 1732648561012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:01,016 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:01,016 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:01,016 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:01,016 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:01,016 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:01,016 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:01,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53568 deadline: 1732648561012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:01,016 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:01,017 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:01,017 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:01,017 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:01,114 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:01,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53558 deadline: 1732648561112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:01,114 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:01,114 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:01,114 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:01,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-26T19:15:01,192 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T19:15:01,192 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T19:15:01,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-11-26T19:15:01,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-26T19:15:01,194 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T19:15:01,195 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T19:15:01,195 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T19:15:01,225 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:01,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53632 deadline: 1732648561222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:01,225 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:01,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53556 deadline: 1732648561222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:01,226 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:01,226 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:01,226 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:01,226 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:01,226 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:01,226 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:01,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53568 deadline: 1732648561222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:01,226 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:01,226 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:01,226 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:01,226 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:01,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-26T19:15:01,315 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/B/0f1594892dbe429b947e32e365b37685 2024-11-26T19:15:01,324 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/C/d26df17321564eca93704812994d6503 is 50, key is test_row_0/C:col10/1732648500485/Put/seqid=0 2024-11-26T19:15:01,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742131_1307 (size=12301) 2024-11-26T19:15:01,347 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-26T19:15:01,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. 2024-11-26T19:15:01,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. as already flushing 2024-11-26T19:15:01,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. 2024-11-26T19:15:01,347 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:01,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:01,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:01,500 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-26T19:15:01,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. 2024-11-26T19:15:01,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. as already flushing 2024-11-26T19:15:01,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. 2024-11-26T19:15:01,501 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:01,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:01,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:01,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-26T19:15:01,536 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:01,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53632 deadline: 1732648561532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:01,536 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:01,536 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:01,536 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:01,536 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:01,536 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:01,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53556 deadline: 1732648561532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:01,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53568 deadline: 1732648561532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:01,537 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:01,537 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:01,537 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:01,537 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:01,537 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:01,537 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:01,625 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:01,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53558 deadline: 1732648561622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:01,626 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:01,626 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:01,626 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:01,653 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-26T19:15:01,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. 2024-11-26T19:15:01,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. as already flushing 2024-11-26T19:15:01,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. 2024-11-26T19:15:01,654 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:01,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:01,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:01,729 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/C/d26df17321564eca93704812994d6503 2024-11-26T19:15:01,738 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/A/ef8afb7318294433b9f12792fa861007 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/A/ef8afb7318294433b9f12792fa861007 2024-11-26T19:15:01,742 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/A/ef8afb7318294433b9f12792fa861007, entries=200, sequenceid=286, filesize=14.4 K 2024-11-26T19:15:01,743 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/B/0f1594892dbe429b947e32e365b37685 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/B/0f1594892dbe429b947e32e365b37685 2024-11-26T19:15:01,747 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/B/0f1594892dbe429b947e32e365b37685, entries=150, sequenceid=286, filesize=12.0 K 2024-11-26T19:15:01,748 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/C/d26df17321564eca93704812994d6503 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/C/d26df17321564eca93704812994d6503 2024-11-26T19:15:01,751 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/C/d26df17321564eca93704812994d6503, entries=150, sequenceid=286, filesize=12.0 K 2024-11-26T19:15:01,752 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~419.95 KB/430032, currentSize=33.54 KB/34350 for 9fea48dabba44dd37de365711fba859f in 867ms, sequenceid=286, compaction requested=true 2024-11-26T19:15:01,752 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9fea48dabba44dd37de365711fba859f: 2024-11-26T19:15:01,752 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9fea48dabba44dd37de365711fba859f:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T19:15:01,752 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:15:01,752 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9fea48dabba44dd37de365711fba859f:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T19:15:01,752 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T19:15:01,752 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T19:15:01,752 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9fea48dabba44dd37de365711fba859f:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T19:15:01,753 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T19:15:01,753 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T19:15:01,754 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 56316 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T19:15:01,754 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49096 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T19:15:01,754 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1541): 9fea48dabba44dd37de365711fba859f/A is initiating minor compaction (all files) 2024-11-26T19:15:01,754 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1541): 9fea48dabba44dd37de365711fba859f/B is initiating minor compaction (all files) 2024-11-26T19:15:01,754 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9fea48dabba44dd37de365711fba859f/B in TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. 2024-11-26T19:15:01,754 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9fea48dabba44dd37de365711fba859f/A in TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. 2024-11-26T19:15:01,754 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/A/f15b8d4d44c14dd990ef71f87db6bc4c, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/A/d684c2e66f464eabb838c5dbadc5bbb1, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/A/ed4375d1e1594ead8299fb77e3ea7be3, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/A/ef8afb7318294433b9f12792fa861007] into tmpdir=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp, totalSize=55.0 K 2024-11-26T19:15:01,754 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/B/afa345985f954ec09e440a65816b226d, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/B/8742c686f20c472eb1cd0f7b30a9efd6, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/B/40c09f8563cf4277a86aa84f990367ae, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/B/0f1594892dbe429b947e32e365b37685] into tmpdir=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp, totalSize=47.9 K 2024-11-26T19:15:01,754 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting f15b8d4d44c14dd990ef71f87db6bc4c, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1732648498490 2024-11-26T19:15:01,755 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting afa345985f954ec09e440a65816b226d, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1732648498490 2024-11-26T19:15:01,755 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting d684c2e66f464eabb838c5dbadc5bbb1, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=219, earliestPutTs=1732648499211 2024-11-26T19:15:01,755 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting 8742c686f20c472eb1cd0f7b30a9efd6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=219, earliestPutTs=1732648499211 2024-11-26T19:15:01,755 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting ed4375d1e1594ead8299fb77e3ea7be3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1732648499237 2024-11-26T19:15:01,755 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting 40c09f8563cf4277a86aa84f990367ae, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1732648499237 2024-11-26T19:15:01,755 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting ef8afb7318294433b9f12792fa861007, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1732648500485 2024-11-26T19:15:01,756 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting 0f1594892dbe429b947e32e365b37685, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1732648500485 2024-11-26T19:15:01,765 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9fea48dabba44dd37de365711fba859f#A#compaction#257 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T19:15:01,765 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9fea48dabba44dd37de365711fba859f#B#compaction#258 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T19:15:01,766 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/A/26e896718c38454f8cac0d7dc59f5bd9 is 50, key is test_row_0/A:col10/1732648500485/Put/seqid=0 2024-11-26T19:15:01,766 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/B/3fd4076cfe914cddbfc11bbf47ebe16a is 50, key is test_row_0/B:col10/1732648500485/Put/seqid=0 2024-11-26T19:15:01,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742132_1308 (size=12779) 2024-11-26T19:15:01,776 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/A/26e896718c38454f8cac0d7dc59f5bd9 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/A/26e896718c38454f8cac0d7dc59f5bd9 2024-11-26T19:15:01,781 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9fea48dabba44dd37de365711fba859f/A of 9fea48dabba44dd37de365711fba859f into 26e896718c38454f8cac0d7dc59f5bd9(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T19:15:01,781 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9fea48dabba44dd37de365711fba859f: 2024-11-26T19:15:01,781 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., storeName=9fea48dabba44dd37de365711fba859f/A, priority=12, startTime=1732648501752; duration=0sec 2024-11-26T19:15:01,782 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T19:15:01,782 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9fea48dabba44dd37de365711fba859f:A 2024-11-26T19:15:01,782 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T19:15:01,784 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742133_1309 (size=12779) 2024-11-26T19:15:01,784 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49096 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T19:15:01,784 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1541): 9fea48dabba44dd37de365711fba859f/C is initiating minor compaction (all files) 2024-11-26T19:15:01,784 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9fea48dabba44dd37de365711fba859f/C in TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. 2024-11-26T19:15:01,785 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/C/8d65a7b76005450db2e45cf5365dcb26, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/C/06dfc70f3cde46b39562ce542b6ad0d1, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/C/870540b8ca6d4daa94b00196a17f1e7c, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/C/d26df17321564eca93704812994d6503] into tmpdir=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp, totalSize=47.9 K 2024-11-26T19:15:01,785 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8d65a7b76005450db2e45cf5365dcb26, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1732648498490 2024-11-26T19:15:01,785 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting 06dfc70f3cde46b39562ce542b6ad0d1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=219, earliestPutTs=1732648499211 2024-11-26T19:15:01,786 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting 870540b8ca6d4daa94b00196a17f1e7c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1732648499237 2024-11-26T19:15:01,787 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting d26df17321564eca93704812994d6503, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1732648500485 2024-11-26T19:15:01,798 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9fea48dabba44dd37de365711fba859f#C#compaction#259 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T19:15:01,799 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/C/2219ca97b35c4801b8d2244c3a103e6b is 50, key is test_row_0/C:col10/1732648500485/Put/seqid=0 2024-11-26T19:15:01,806 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-26T19:15:01,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. 2024-11-26T19:15:01,807 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing 9fea48dabba44dd37de365711fba859f 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-26T19:15:01,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9fea48dabba44dd37de365711fba859f, store=A 2024-11-26T19:15:01,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:15:01,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9fea48dabba44dd37de365711fba859f, store=B 2024-11-26T19:15:01,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:15:01,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9fea48dabba44dd37de365711fba859f, store=C 2024-11-26T19:15:01,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:15:01,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-26T19:15:01,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/A/4e8ca4cc827749c7b3aceb2286ffcfe7 is 50, key is test_row_0/A:col10/1732648500898/Put/seqid=0 2024-11-26T19:15:01,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742134_1310 (size=12779) 2024-11-26T19:15:01,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742135_1311 (size=12301) 2024-11-26T19:15:01,830 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=295 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/A/4e8ca4cc827749c7b3aceb2286ffcfe7 2024-11-26T19:15:01,833 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/C/2219ca97b35c4801b8d2244c3a103e6b as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/C/2219ca97b35c4801b8d2244c3a103e6b 2024-11-26T19:15:01,843 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9fea48dabba44dd37de365711fba859f/C of 9fea48dabba44dd37de365711fba859f into 2219ca97b35c4801b8d2244c3a103e6b(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T19:15:01,843 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9fea48dabba44dd37de365711fba859f: 2024-11-26T19:15:01,843 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., storeName=9fea48dabba44dd37de365711fba859f/C, priority=12, startTime=1732648501752; duration=0sec 2024-11-26T19:15:01,843 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:15:01,843 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9fea48dabba44dd37de365711fba859f:C 2024-11-26T19:15:01,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/B/72c0dc24c74e47ebb78105ddf4b18417 is 50, key is test_row_0/B:col10/1732648500898/Put/seqid=0 2024-11-26T19:15:01,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742136_1312 (size=12301) 2024-11-26T19:15:01,851 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=295 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/B/72c0dc24c74e47ebb78105ddf4b18417 2024-11-26T19:15:01,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/C/5577393f361b45f989c7c7cb5e5e52ec is 50, key is test_row_0/C:col10/1732648500898/Put/seqid=0 2024-11-26T19:15:01,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742137_1313 (size=12301) 2024-11-26T19:15:01,864 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=295 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/C/5577393f361b45f989c7c7cb5e5e52ec 2024-11-26T19:15:01,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/A/4e8ca4cc827749c7b3aceb2286ffcfe7 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/A/4e8ca4cc827749c7b3aceb2286ffcfe7 2024-11-26T19:15:01,874 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/A/4e8ca4cc827749c7b3aceb2286ffcfe7, entries=150, sequenceid=295, filesize=12.0 K 2024-11-26T19:15:01,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/B/72c0dc24c74e47ebb78105ddf4b18417 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/B/72c0dc24c74e47ebb78105ddf4b18417 2024-11-26T19:15:01,880 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/B/72c0dc24c74e47ebb78105ddf4b18417, entries=150, sequenceid=295, filesize=12.0 K 2024-11-26T19:15:01,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/C/5577393f361b45f989c7c7cb5e5e52ec as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/C/5577393f361b45f989c7c7cb5e5e52ec 2024-11-26T19:15:01,886 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/C/5577393f361b45f989c7c7cb5e5e52ec, entries=150, sequenceid=295, filesize=12.0 K 2024-11-26T19:15:01,886 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 9fea48dabba44dd37de365711fba859f in 79ms, sequenceid=295, compaction requested=false 2024-11-26T19:15:01,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for 9fea48dabba44dd37de365711fba859f: 2024-11-26T19:15:01,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. 2024-11-26T19:15:01,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-11-26T19:15:01,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-11-26T19:15:01,891 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-26T19:15:01,891 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 693 msec 2024-11-26T19:15:01,893 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 700 msec 2024-11-26T19:15:02,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(8855): Flush requested on 9fea48dabba44dd37de365711fba859f 2024-11-26T19:15:02,052 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9fea48dabba44dd37de365711fba859f 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-26T19:15:02,053 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9fea48dabba44dd37de365711fba859f, store=A 2024-11-26T19:15:02,053 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:15:02,053 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9fea48dabba44dd37de365711fba859f, store=B 2024-11-26T19:15:02,053 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:15:02,053 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9fea48dabba44dd37de365711fba859f, store=C 2024-11-26T19:15:02,053 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:15:02,058 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/A/e0e1ec2b7a014376bf4101ad46c745ba is 50, key is test_row_0/A:col10/1732648502051/Put/seqid=0 2024-11-26T19:15:02,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742138_1314 (size=14741) 2024-11-26T19:15:02,125 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:02,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53568 deadline: 1732648562119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:02,126 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:02,126 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:02,127 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:02,127 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732648502115, max timestamp=1732648502124Num uniques 859; 2024-11-26T19:15:02,127 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-26T19:15:02,128 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732648502115, max timestamp=1732648502124Num uniques 859; 2024-11-26T19:15:02,128 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732648502115, max timestamp=1732648502124Num uniques 859; 2024-11-26T19:15:02,128 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-26T19:15:02,128 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-26T19:15:02,178 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:02,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53556 deadline: 1732648562173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:02,179 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:02,179 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:02,179 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:02,180 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:02,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53632 deadline: 1732648562174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:02,181 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:02,181 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:02,181 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:02,189 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/B/3fd4076cfe914cddbfc11bbf47ebe16a as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/B/3fd4076cfe914cddbfc11bbf47ebe16a 2024-11-26T19:15:02,193 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9fea48dabba44dd37de365711fba859f/B of 9fea48dabba44dd37de365711fba859f into 3fd4076cfe914cddbfc11bbf47ebe16a(size=12.5 K), total size for store is 24.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T19:15:02,193 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9fea48dabba44dd37de365711fba859f: 2024-11-26T19:15:02,193 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., storeName=9fea48dabba44dd37de365711fba859f/B, priority=12, startTime=1732648501752; duration=0sec 2024-11-26T19:15:02,193 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:15:02,193 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9fea48dabba44dd37de365711fba859f:B 2024-11-26T19:15:02,235 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:02,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53568 deadline: 1732648562232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:02,235 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:02,235 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:02,235 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:02,286 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:02,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53556 deadline: 1732648562282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:02,286 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:02,286 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:02,286 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:02,296 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:02,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53632 deadline: 1732648562292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:02,296 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:02,296 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:02,296 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:02,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-26T19:15:02,332 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T19:15:02,332 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T19:15:02,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-11-26T19:15:02,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-26T19:15:02,334 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T19:15:02,335 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T19:15:02,335 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T19:15:02,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-26T19:15:02,443 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:02,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53568 deadline: 1732648562441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:02,443 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:02,443 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:02,443 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:02,466 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=307 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/A/e0e1ec2b7a014376bf4101ad46c745ba 2024-11-26T19:15:02,474 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/B/e673290407d2424b8f67ea4c4af8c9ab is 50, key is test_row_0/B:col10/1732648502051/Put/seqid=0 2024-11-26T19:15:02,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742139_1315 (size=12301) 2024-11-26T19:15:02,486 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-26T19:15:02,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. 2024-11-26T19:15:02,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. as already flushing 2024-11-26T19:15:02,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. 2024-11-26T19:15:02,487 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:02,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:02,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:02,495 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:02,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53556 deadline: 1732648562492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:02,496 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:02,496 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:02,496 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:02,506 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:02,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53632 deadline: 1732648562502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:02,507 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:02,507 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:02,507 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:02,631 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:02,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53558 deadline: 1732648562631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:02,632 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to fe76acf732b2,42785,1732648423970 for 'test_row_1' in 9fea48dabba44dd37de365711fba859f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2159 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T19:15:02,632 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:02,632 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:02,633 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:02,639 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-26T19:15:02,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. 2024-11-26T19:15:02,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. as already flushing 2024-11-26T19:15:02,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. 2024-11-26T19:15:02,640 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:02,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:02,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:02,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-26T19:15:02,754 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:02,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53568 deadline: 1732648562751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:02,754 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:02,754 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:02,754 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:02,793 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-26T19:15:02,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. 2024-11-26T19:15:02,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. as already flushing 2024-11-26T19:15:02,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. 2024-11-26T19:15:02,793 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:02,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:02,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:02,805 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:02,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53556 deadline: 1732648562802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:02,806 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:02,806 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:02,806 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:02,816 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:02,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53632 deadline: 1732648562812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:02,816 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:02,816 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:02,816 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:02,879 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=307 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/B/e673290407d2424b8f67ea4c4af8c9ab 2024-11-26T19:15:02,886 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/C/f972aaf11a6140dfa1bc08bdbdfbe26b is 50, key is test_row_0/C:col10/1732648502051/Put/seqid=0 2024-11-26T19:15:02,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742140_1316 (size=12301) 2024-11-26T19:15:02,946 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-26T19:15:02,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. 2024-11-26T19:15:02,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. as already flushing 2024-11-26T19:15:02,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. 2024-11-26T19:15:02,947 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:02,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:02,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:02,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-26T19:15:03,099 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-26T19:15:03,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. 2024-11-26T19:15:03,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. as already flushing 2024-11-26T19:15:03,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. 2024-11-26T19:15:03,100 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:03,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:03,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:03,253 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-26T19:15:03,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. 2024-11-26T19:15:03,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. as already flushing 2024-11-26T19:15:03,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. 2024-11-26T19:15:03,253 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:03,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:03,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:03,266 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:03,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53568 deadline: 1732648563262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:03,267 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:03,267 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:03,267 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:03,291 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=307 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/C/f972aaf11a6140dfa1bc08bdbdfbe26b 2024-11-26T19:15:03,295 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/A/e0e1ec2b7a014376bf4101ad46c745ba as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/A/e0e1ec2b7a014376bf4101ad46c745ba 2024-11-26T19:15:03,306 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/A/e0e1ec2b7a014376bf4101ad46c745ba, entries=200, sequenceid=307, filesize=14.4 K 2024-11-26T19:15:03,306 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/B/e673290407d2424b8f67ea4c4af8c9ab as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/B/e673290407d2424b8f67ea4c4af8c9ab 2024-11-26T19:15:03,311 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/B/e673290407d2424b8f67ea4c4af8c9ab, entries=150, sequenceid=307, filesize=12.0 K 2024-11-26T19:15:03,311 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/C/f972aaf11a6140dfa1bc08bdbdfbe26b as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/C/f972aaf11a6140dfa1bc08bdbdfbe26b 2024-11-26T19:15:03,315 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/C/f972aaf11a6140dfa1bc08bdbdfbe26b, entries=150, sequenceid=307, filesize=12.0 K 2024-11-26T19:15:03,316 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 9fea48dabba44dd37de365711fba859f in 1265ms, sequenceid=307, compaction requested=true 2024-11-26T19:15:03,316 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9fea48dabba44dd37de365711fba859f: 2024-11-26T19:15:03,316 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9fea48dabba44dd37de365711fba859f:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T19:15:03,316 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:15:03,316 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T19:15:03,316 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9fea48dabba44dd37de365711fba859f:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T19:15:03,316 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:15:03,316 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9fea48dabba44dd37de365711fba859f:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T19:15:03,316 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T19:15:03,316 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T19:15:03,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(8855): Flush requested on 9fea48dabba44dd37de365711fba859f 2024-11-26T19:15:03,317 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39821 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T19:15:03,317 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9fea48dabba44dd37de365711fba859f 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-26T19:15:03,317 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1541): 9fea48dabba44dd37de365711fba859f/A is initiating minor compaction (all files) 2024-11-26T19:15:03,317 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9fea48dabba44dd37de365711fba859f/A in TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. 2024-11-26T19:15:03,317 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/A/26e896718c38454f8cac0d7dc59f5bd9, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/A/4e8ca4cc827749c7b3aceb2286ffcfe7, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/A/e0e1ec2b7a014376bf4101ad46c745ba] into tmpdir=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp, totalSize=38.9 K 2024-11-26T19:15:03,318 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9fea48dabba44dd37de365711fba859f, store=A 2024-11-26T19:15:03,318 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37381 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T19:15:03,318 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting 26e896718c38454f8cac0d7dc59f5bd9, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1732648500485 2024-11-26T19:15:03,318 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1541): 9fea48dabba44dd37de365711fba859f/B is initiating minor compaction (all files) 2024-11-26T19:15:03,318 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9fea48dabba44dd37de365711fba859f/B in TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. 2024-11-26T19:15:03,318 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/B/3fd4076cfe914cddbfc11bbf47ebe16a, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/B/72c0dc24c74e47ebb78105ddf4b18417, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/B/e673290407d2424b8f67ea4c4af8c9ab] into tmpdir=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp, totalSize=36.5 K 2024-11-26T19:15:03,318 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4e8ca4cc827749c7b3aceb2286ffcfe7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1732648500886 2024-11-26T19:15:03,318 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting 3fd4076cfe914cddbfc11bbf47ebe16a, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1732648500485 2024-11-26T19:15:03,319 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting e0e1ec2b7a014376bf4101ad46c745ba, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1732648502047 2024-11-26T19:15:03,319 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T19:15:03,319 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9fea48dabba44dd37de365711fba859f, store=B 2024-11-26T19:15:03,319 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T19:15:03,319 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9fea48dabba44dd37de365711fba859f, store=C 2024-11-26T19:15:03,319 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T19:15:03,319 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting 72c0dc24c74e47ebb78105ddf4b18417, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1732648500886 2024-11-26T19:15:03,319 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting e673290407d2424b8f67ea4c4af8c9ab, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1732648502048 2024-11-26T19:15:03,330 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9fea48dabba44dd37de365711fba859f#B#compaction#267 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T19:15:03,331 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9fea48dabba44dd37de365711fba859f#A#compaction#266 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T19:15:03,331 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/B/c618467f02014536825687fe8d6242ad is 50, key is test_row_0/B:col10/1732648502051/Put/seqid=0 2024-11-26T19:15:03,331 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/A/a1955113f8284fe1a4c3ee5461f76cf7 is 50, key is test_row_0/A:col10/1732648502051/Put/seqid=0 2024-11-26T19:15:03,337 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/A/aa4d10218d08424c9fe8bd9037543bd1 is 50, key is test_row_0/A:col10/1732648502140/Put/seqid=0 2024-11-26T19:15:03,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742141_1317 (size=12881) 2024-11-26T19:15:03,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742142_1318 (size=12881) 2024-11-26T19:15:03,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742143_1319 (size=12301) 2024-11-26T19:15:03,348 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=344 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/A/aa4d10218d08424c9fe8bd9037543bd1 2024-11-26T19:15:03,352 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:03,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53556 deadline: 1732648563346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:03,353 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:03,353 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:03,353 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:03,354 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/B/475942e1b59f4b6ba54430773423a0cb is 50, key is test_row_0/B:col10/1732648502140/Put/seqid=0 2024-11-26T19:15:03,359 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:03,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53632 deadline: 1732648563352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:03,360 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:03,360 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:03,360 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:03,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742144_1320 (size=12301) 2024-11-26T19:15:03,406 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-26T19:15:03,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. 2024-11-26T19:15:03,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. as already flushing 2024-11-26T19:15:03,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. 2024-11-26T19:15:03,407 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:03,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:03,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:03,466 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:03,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53556 deadline: 1732648563462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:03,467 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:03,467 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:03,467 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:03,467 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:03,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53632 deadline: 1732648563462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:03,467 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:03,467 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:03,467 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:03,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-26T19:15:03,559 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-26T19:15:03,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. 2024-11-26T19:15:03,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. as already flushing 2024-11-26T19:15:03,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. 2024-11-26T19:15:03,560 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:03,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:03,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:03,675 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:03,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53556 deadline: 1732648563671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:03,676 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:03,676 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:03,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53632 deadline: 1732648563671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:03,676 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:03,676 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:03,676 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:03,676 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:03,676 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:03,712 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-26T19:15:03,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. 2024-11-26T19:15:03,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. as already flushing 2024-11-26T19:15:03,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. 2024-11-26T19:15:03,713 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:03,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:03,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:03,745 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/B/c618467f02014536825687fe8d6242ad as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/B/c618467f02014536825687fe8d6242ad 2024-11-26T19:15:03,747 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/A/a1955113f8284fe1a4c3ee5461f76cf7 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/A/a1955113f8284fe1a4c3ee5461f76cf7 2024-11-26T19:15:03,750 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9fea48dabba44dd37de365711fba859f/B of 9fea48dabba44dd37de365711fba859f into c618467f02014536825687fe8d6242ad(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T19:15:03,750 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9fea48dabba44dd37de365711fba859f: 2024-11-26T19:15:03,750 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., storeName=9fea48dabba44dd37de365711fba859f/B, priority=13, startTime=1732648503316; duration=0sec 2024-11-26T19:15:03,750 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T19:15:03,750 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9fea48dabba44dd37de365711fba859f:B 2024-11-26T19:15:03,750 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T19:15:03,751 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37381 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T19:15:03,751 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1541): 9fea48dabba44dd37de365711fba859f/C is initiating minor compaction (all files) 2024-11-26T19:15:03,751 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9fea48dabba44dd37de365711fba859f/C in TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. 2024-11-26T19:15:03,752 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/C/2219ca97b35c4801b8d2244c3a103e6b, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/C/5577393f361b45f989c7c7cb5e5e52ec, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/C/f972aaf11a6140dfa1bc08bdbdfbe26b] into tmpdir=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp, totalSize=36.5 K 2024-11-26T19:15:03,752 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9fea48dabba44dd37de365711fba859f/A of 9fea48dabba44dd37de365711fba859f into a1955113f8284fe1a4c3ee5461f76cf7(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T19:15:03,752 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting 2219ca97b35c4801b8d2244c3a103e6b, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1732648500485 2024-11-26T19:15:03,752 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9fea48dabba44dd37de365711fba859f: 2024-11-26T19:15:03,752 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., storeName=9fea48dabba44dd37de365711fba859f/A, priority=13, startTime=1732648503316; duration=0sec 2024-11-26T19:15:03,753 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:15:03,753 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9fea48dabba44dd37de365711fba859f:A 2024-11-26T19:15:03,753 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting 5577393f361b45f989c7c7cb5e5e52ec, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1732648500886 2024-11-26T19:15:03,753 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting f972aaf11a6140dfa1bc08bdbdfbe26b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1732648502048 2024-11-26T19:15:03,768 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=344 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/B/475942e1b59f4b6ba54430773423a0cb 2024-11-26T19:15:03,776 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/C/ea88edf691d54926836dbbd48ba0f773 is 50, key is test_row_0/C:col10/1732648502140/Put/seqid=0 2024-11-26T19:15:03,776 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9fea48dabba44dd37de365711fba859f#C#compaction#271 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T19:15:03,777 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/C/989c1bef759440419fbd0b43bed9b6cb is 50, key is test_row_0/C:col10/1732648502051/Put/seqid=0 2024-11-26T19:15:03,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742145_1321 (size=12301) 2024-11-26T19:15:03,789 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=344 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/C/ea88edf691d54926836dbbd48ba0f773 2024-11-26T19:15:03,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742146_1322 (size=12881) 2024-11-26T19:15:03,794 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/A/aa4d10218d08424c9fe8bd9037543bd1 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/A/aa4d10218d08424c9fe8bd9037543bd1 2024-11-26T19:15:03,796 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/C/989c1bef759440419fbd0b43bed9b6cb as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/C/989c1bef759440419fbd0b43bed9b6cb 2024-11-26T19:15:03,799 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/A/aa4d10218d08424c9fe8bd9037543bd1, entries=150, sequenceid=344, filesize=12.0 K 2024-11-26T19:15:03,800 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/B/475942e1b59f4b6ba54430773423a0cb as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/B/475942e1b59f4b6ba54430773423a0cb 2024-11-26T19:15:03,803 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9fea48dabba44dd37de365711fba859f/C of 9fea48dabba44dd37de365711fba859f into 989c1bef759440419fbd0b43bed9b6cb(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T19:15:03,803 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9fea48dabba44dd37de365711fba859f: 2024-11-26T19:15:03,803 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., storeName=9fea48dabba44dd37de365711fba859f/C, priority=13, startTime=1732648503316; duration=0sec 2024-11-26T19:15:03,804 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:15:03,804 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9fea48dabba44dd37de365711fba859f:C 2024-11-26T19:15:03,807 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/B/475942e1b59f4b6ba54430773423a0cb, entries=150, sequenceid=344, filesize=12.0 K 2024-11-26T19:15:03,808 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/C/ea88edf691d54926836dbbd48ba0f773 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/C/ea88edf691d54926836dbbd48ba0f773 2024-11-26T19:15:03,812 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/C/ea88edf691d54926836dbbd48ba0f773, entries=150, sequenceid=344, filesize=12.0 K 2024-11-26T19:15:03,813 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 9fea48dabba44dd37de365711fba859f in 496ms, sequenceid=344, compaction requested=false 2024-11-26T19:15:03,813 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9fea48dabba44dd37de365711fba859f: 2024-11-26T19:15:03,866 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-26T19:15:03,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. 2024-11-26T19:15:03,866 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing 9fea48dabba44dd37de365711fba859f 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-26T19:15:03,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9fea48dabba44dd37de365711fba859f, store=A 2024-11-26T19:15:03,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:15:03,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9fea48dabba44dd37de365711fba859f, store=B 2024-11-26T19:15:03,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:15:03,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9fea48dabba44dd37de365711fba859f, store=C 2024-11-26T19:15:03,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:15:03,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/A/6e6c0ae7f2674e1f92544ced5ab45597 is 50, key is test_row_0/A:col10/1732648503336/Put/seqid=0 2024-11-26T19:15:03,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742147_1323 (size=12301) 2024-11-26T19:15:03,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(8855): Flush requested on 9fea48dabba44dd37de365711fba859f 2024-11-26T19:15:03,985 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. as already flushing 2024-11-26T19:15:04,036 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732648503997, max timestamp=1732648504032Num uniques 859; 2024-11-26T19:15:04,036 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732648503997, max timestamp=1732648504032Num uniques 859; 2024-11-26T19:15:04,036 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-26T19:15:04,036 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-26T19:15:04,036 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732648503997, max timestamp=1732648504032Num uniques 859; 2024-11-26T19:15:04,036 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-26T19:15:04,091 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:04,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53632 deadline: 1732648564086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:04,091 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:04,091 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:04,091 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:04,094 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:04,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53556 deadline: 1732648564089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:04,095 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:04,095 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:04,095 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:04,206 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:04,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53632 deadline: 1732648564202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:04,206 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:04,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53556 deadline: 1732648564202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:04,206 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:04,207 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:04,207 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:04,207 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:04,207 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:04,207 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:04,277 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=357 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/A/6e6c0ae7f2674e1f92544ced5ab45597 2024-11-26T19:15:04,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/B/36a8189df54f408ba4c725468ef7f3f6 is 50, key is test_row_0/B:col10/1732648503336/Put/seqid=0 2024-11-26T19:15:04,286 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:04,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53568 deadline: 1732648564282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:04,286 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to fe76acf732b2,42785,1732648423970 for 'test_row_2' in 9fea48dabba44dd37de365711fba859f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T19:15:04,287 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:04,287 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:04,287 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:04,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742148_1324 (size=12301) 2024-11-26T19:15:04,416 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:04,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53632 deadline: 1732648564412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:04,417 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:04,417 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:04,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53556 deadline: 1732648564412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:04,417 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:04,417 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:04,417 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:04,417 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:04,417 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:04,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-26T19:15:04,657 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:04,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53558 deadline: 1732648564652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:04,657 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to fe76acf732b2,42785,1732648423970 for 'test_row_1' in 9fea48dabba44dd37de365711fba859f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4184 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T19:15:04,658 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:04,658 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:04,658 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:04,696 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=357 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/B/36a8189df54f408ba4c725468ef7f3f6 2024-11-26T19:15:04,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/C/ea2759e5f801454cb6a5f87d75121ee2 is 50, key is test_row_0/C:col10/1732648503336/Put/seqid=0 2024-11-26T19:15:04,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742149_1325 (size=12301) 2024-11-26T19:15:04,726 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:04,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53632 deadline: 1732648564721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:04,726 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:04,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53556 deadline: 1732648564722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:04,726 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:04,726 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:04,726 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:04,727 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:04,727 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:04,727 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:05,107 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=357 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/C/ea2759e5f801454cb6a5f87d75121ee2 2024-11-26T19:15:05,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/A/6e6c0ae7f2674e1f92544ced5ab45597 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/A/6e6c0ae7f2674e1f92544ced5ab45597 2024-11-26T19:15:05,116 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/A/6e6c0ae7f2674e1f92544ced5ab45597, entries=150, sequenceid=357, filesize=12.0 K 2024-11-26T19:15:05,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/B/36a8189df54f408ba4c725468ef7f3f6 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/B/36a8189df54f408ba4c725468ef7f3f6 2024-11-26T19:15:05,121 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/B/36a8189df54f408ba4c725468ef7f3f6, entries=150, sequenceid=357, filesize=12.0 K 2024-11-26T19:15:05,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/C/ea2759e5f801454cb6a5f87d75121ee2 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/C/ea2759e5f801454cb6a5f87d75121ee2 2024-11-26T19:15:05,124 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/C/ea2759e5f801454cb6a5f87d75121ee2, entries=150, sequenceid=357, filesize=12.0 K 2024-11-26T19:15:05,125 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 9fea48dabba44dd37de365711fba859f in 1259ms, sequenceid=357, compaction requested=true 2024-11-26T19:15:05,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for 9fea48dabba44dd37de365711fba859f: 2024-11-26T19:15:05,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. 2024-11-26T19:15:05,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-11-26T19:15:05,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-11-26T19:15:05,128 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-26T19:15:05,128 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7920 sec 2024-11-26T19:15:05,130 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 2.7970 sec 2024-11-26T19:15:05,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(8855): Flush requested on 9fea48dabba44dd37de365711fba859f 2024-11-26T19:15:05,237 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9fea48dabba44dd37de365711fba859f 3/3 column families, dataSize=140.89 KB heapSize=400.45 KB 2024-11-26T19:15:05,240 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732648504069, max timestamp=1732648505236Num uniques 1068; 2024-11-26T19:15:05,240 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732648504069, max timestamp=1732648505236Num uniques 1068; 2024-11-26T19:15:05,241 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-26T19:15:05,241 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-26T19:15:05,241 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9fea48dabba44dd37de365711fba859f, store=A 2024-11-26T19:15:05,242 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T19:15:05,242 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9fea48dabba44dd37de365711fba859f, store=B 2024-11-26T19:15:05,242 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T19:15:05,242 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9fea48dabba44dd37de365711fba859f, store=C 2024-11-26T19:15:05,242 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-26T19:15:05,246 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/A/d268b2561d2d46bf955f8f2c8c5b3154 is 50, key is test_row_0/A:col10/1732648504069/Put/seqid=0 2024-11-26T19:15:05,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742150_1326 (size=12301) 2024-11-26T19:15:05,257 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=395 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/A/d268b2561d2d46bf955f8f2c8c5b3154 2024-11-26T19:15:05,265 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/B/ac28fcfe39004e69acb9e67d5431c1fd is 50, key is test_row_0/B:col10/1732648504069/Put/seqid=0 2024-11-26T19:15:05,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742151_1327 (size=12301) 2024-11-26T19:15:05,306 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:05,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53632 deadline: 1732648565301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:05,306 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:05,306 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:05,306 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:05,310 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:05,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53556 deadline: 1732648565306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:05,311 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:05,311 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:05,311 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:05,415 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:05,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53632 deadline: 1732648565412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:05,416 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:05,416 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:05,416 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:05,424 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:05,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53556 deadline: 1732648565422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:05,424 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:05,424 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:05,424 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:05,624 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:05,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53632 deadline: 1732648565622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:05,625 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:05,625 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:05,625 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:05,635 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:05,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53556 deadline: 1732648565632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:05,635 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:05,635 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:05,635 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:05,678 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=395 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/B/ac28fcfe39004e69acb9e67d5431c1fd 2024-11-26T19:15:05,686 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/C/6b282d4f0bca438189eb2b19645b1717 is 50, key is test_row_0/C:col10/1732648504069/Put/seqid=0 2024-11-26T19:15:05,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742152_1328 (size=12301) 2024-11-26T19:15:05,934 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:05,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53632 deadline: 1732648565931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:05,934 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:05,934 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:05,935 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:05,944 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:05,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53556 deadline: 1732648565942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:05,945 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:05,945 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:05,945 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:06,091 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=395 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/C/6b282d4f0bca438189eb2b19645b1717 2024-11-26T19:15:06,096 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/A/d268b2561d2d46bf955f8f2c8c5b3154 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/A/d268b2561d2d46bf955f8f2c8c5b3154 2024-11-26T19:15:06,099 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/A/d268b2561d2d46bf955f8f2c8c5b3154, entries=150, sequenceid=395, filesize=12.0 K 2024-11-26T19:15:06,100 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/B/ac28fcfe39004e69acb9e67d5431c1fd as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/B/ac28fcfe39004e69acb9e67d5431c1fd 2024-11-26T19:15:06,104 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/B/ac28fcfe39004e69acb9e67d5431c1fd, entries=150, sequenceid=395, filesize=12.0 K 2024-11-26T19:15:06,105 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/C/6b282d4f0bca438189eb2b19645b1717 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/C/6b282d4f0bca438189eb2b19645b1717 2024-11-26T19:15:06,109 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/C/6b282d4f0bca438189eb2b19645b1717, entries=150, sequenceid=395, filesize=12.0 K 2024-11-26T19:15:06,110 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~183.83 KB/188240, currentSize=114.05 KB/116790 for 9fea48dabba44dd37de365711fba859f in 873ms, sequenceid=395, compaction requested=true 2024-11-26T19:15:06,110 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9fea48dabba44dd37de365711fba859f: 2024-11-26T19:15:06,110 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9fea48dabba44dd37de365711fba859f:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T19:15:06,110 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:15:06,110 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9fea48dabba44dd37de365711fba859f:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T19:15:06,110 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T19:15:06,110 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:15:06,110 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9fea48dabba44dd37de365711fba859f:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T19:15:06,110 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T19:15:06,110 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T19:15:06,111 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49784 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T19:15:06,111 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1541): 9fea48dabba44dd37de365711fba859f/A is initiating minor compaction (all files) 2024-11-26T19:15:06,111 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49784 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T19:15:06,111 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9fea48dabba44dd37de365711fba859f/A in TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. 2024-11-26T19:15:06,111 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1541): 9fea48dabba44dd37de365711fba859f/B is initiating minor compaction (all files) 2024-11-26T19:15:06,111 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/A/a1955113f8284fe1a4c3ee5461f76cf7, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/A/aa4d10218d08424c9fe8bd9037543bd1, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/A/6e6c0ae7f2674e1f92544ced5ab45597, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/A/d268b2561d2d46bf955f8f2c8c5b3154] into tmpdir=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp, totalSize=48.6 K 2024-11-26T19:15:06,111 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9fea48dabba44dd37de365711fba859f/B in TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. 2024-11-26T19:15:06,111 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/B/c618467f02014536825687fe8d6242ad, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/B/475942e1b59f4b6ba54430773423a0cb, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/B/36a8189df54f408ba4c725468ef7f3f6, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/B/ac28fcfe39004e69acb9e67d5431c1fd] into tmpdir=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp, totalSize=48.6 K 2024-11-26T19:15:06,112 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting a1955113f8284fe1a4c3ee5461f76cf7, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1732648502048 2024-11-26T19:15:06,112 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting c618467f02014536825687fe8d6242ad, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1732648502048 2024-11-26T19:15:06,112 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting aa4d10218d08424c9fe8bd9037543bd1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1732648502140 2024-11-26T19:15:06,112 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting 475942e1b59f4b6ba54430773423a0cb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1732648502140 2024-11-26T19:15:06,112 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6e6c0ae7f2674e1f92544ced5ab45597, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1732648503336 2024-11-26T19:15:06,112 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting 36a8189df54f408ba4c725468ef7f3f6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1732648503336 2024-11-26T19:15:06,113 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting d268b2561d2d46bf955f8f2c8c5b3154, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1732648504069 2024-11-26T19:15:06,113 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting ac28fcfe39004e69acb9e67d5431c1fd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1732648504069 2024-11-26T19:15:06,122 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9fea48dabba44dd37de365711fba859f#A#compaction#278 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T19:15:06,122 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/A/01b385969a024104943b10b35fcbfa9d is 50, key is test_row_0/A:col10/1732648504069/Put/seqid=0 2024-11-26T19:15:06,125 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9fea48dabba44dd37de365711fba859f#B#compaction#279 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T19:15:06,126 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/B/bd5a990699814b709ea2a6953e0142e2 is 50, key is test_row_0/B:col10/1732648504069/Put/seqid=0 2024-11-26T19:15:06,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742153_1329 (size=13017) 2024-11-26T19:15:06,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742154_1330 (size=13017) 2024-11-26T19:15:06,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(8855): Flush requested on 9fea48dabba44dd37de365711fba859f 2024-11-26T19:15:06,306 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9fea48dabba44dd37de365711fba859f 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-26T19:15:06,306 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9fea48dabba44dd37de365711fba859f, store=A 2024-11-26T19:15:06,306 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T19:15:06,306 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9fea48dabba44dd37de365711fba859f, store=B 2024-11-26T19:15:06,306 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T19:15:06,306 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9fea48dabba44dd37de365711fba859f, store=C 2024-11-26T19:15:06,306 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T19:15:06,313 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/A/2fb434edd91440b2a87edb4496632288 is 50, key is test_row_0/A:col10/1732648505292/Put/seqid=0 2024-11-26T19:15:06,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742155_1331 (size=12301) 2024-11-26T19:15:06,318 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=416 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/A/2fb434edd91440b2a87edb4496632288 2024-11-26T19:15:06,327 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/B/95498f2feaf74bfc82b8cfc30f5cae2d is 50, key is test_row_0/B:col10/1732648505292/Put/seqid=0 2024-11-26T19:15:06,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742156_1332 (size=12301) 2024-11-26T19:15:06,357 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:06,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53568 deadline: 1732648566352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:06,358 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:06,358 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:06,358 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:06,445 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:06,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53632 deadline: 1732648566441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:06,445 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:06,445 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:06,445 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:06,456 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:06,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53556 deadline: 1732648566451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:06,456 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:06,456 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:06,456 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:06,463 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:06,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53568 deadline: 1732648566462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:06,463 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:06,463 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:06,463 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:06,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-26T19:15:06,492 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T19:15:06,493 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T19:15:06,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-11-26T19:15:06,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-26T19:15:06,495 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T19:15:06,495 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T19:15:06,495 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T19:15:06,535 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/A/01b385969a024104943b10b35fcbfa9d as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/A/01b385969a024104943b10b35fcbfa9d 2024-11-26T19:15:06,540 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9fea48dabba44dd37de365711fba859f/A of 9fea48dabba44dd37de365711fba859f into 01b385969a024104943b10b35fcbfa9d(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T19:15:06,540 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9fea48dabba44dd37de365711fba859f: 2024-11-26T19:15:06,540 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., storeName=9fea48dabba44dd37de365711fba859f/A, priority=12, startTime=1732648506110; duration=0sec 2024-11-26T19:15:06,540 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T19:15:06,540 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9fea48dabba44dd37de365711fba859f:A 2024-11-26T19:15:06,540 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T19:15:06,541 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49784 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T19:15:06,541 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1541): 9fea48dabba44dd37de365711fba859f/C is initiating minor compaction (all files) 2024-11-26T19:15:06,541 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9fea48dabba44dd37de365711fba859f/C in TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. 2024-11-26T19:15:06,541 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/C/989c1bef759440419fbd0b43bed9b6cb, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/C/ea88edf691d54926836dbbd48ba0f773, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/C/ea2759e5f801454cb6a5f87d75121ee2, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/C/6b282d4f0bca438189eb2b19645b1717] into tmpdir=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp, totalSize=48.6 K 2024-11-26T19:15:06,542 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting 989c1bef759440419fbd0b43bed9b6cb, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1732648502048 2024-11-26T19:15:06,542 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting ea88edf691d54926836dbbd48ba0f773, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1732648502140 2024-11-26T19:15:06,543 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting ea2759e5f801454cb6a5f87d75121ee2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1732648503336 2024-11-26T19:15:06,543 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6b282d4f0bca438189eb2b19645b1717, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1732648504069 2024-11-26T19:15:06,547 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/B/bd5a990699814b709ea2a6953e0142e2 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/B/bd5a990699814b709ea2a6953e0142e2 2024-11-26T19:15:06,552 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9fea48dabba44dd37de365711fba859f/B of 9fea48dabba44dd37de365711fba859f into bd5a990699814b709ea2a6953e0142e2(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T19:15:06,552 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9fea48dabba44dd37de365711fba859f: 2024-11-26T19:15:06,552 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., storeName=9fea48dabba44dd37de365711fba859f/B, priority=12, startTime=1732648506110; duration=0sec 2024-11-26T19:15:06,552 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:15:06,552 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9fea48dabba44dd37de365711fba859f:B 2024-11-26T19:15:06,554 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9fea48dabba44dd37de365711fba859f#C#compaction#282 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T19:15:06,554 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/C/95a8a434e53e48d0b5b2f0c54c3e114d is 50, key is test_row_0/C:col10/1732648504069/Put/seqid=0 2024-11-26T19:15:06,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742157_1333 (size=13017) 2024-11-26T19:15:06,563 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/C/95a8a434e53e48d0b5b2f0c54c3e114d as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/C/95a8a434e53e48d0b5b2f0c54c3e114d 2024-11-26T19:15:06,567 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9fea48dabba44dd37de365711fba859f/C of 9fea48dabba44dd37de365711fba859f into 95a8a434e53e48d0b5b2f0c54c3e114d(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T19:15:06,567 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9fea48dabba44dd37de365711fba859f: 2024-11-26T19:15:06,568 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., storeName=9fea48dabba44dd37de365711fba859f/C, priority=12, startTime=1732648506110; duration=0sec 2024-11-26T19:15:06,568 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:15:06,568 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9fea48dabba44dd37de365711fba859f:C 2024-11-26T19:15:06,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-26T19:15:06,647 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-26T19:15:06,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. 2024-11-26T19:15:06,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. as already flushing 2024-11-26T19:15:06,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. 2024-11-26T19:15:06,648 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:06,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:06,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:06,674 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:06,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53568 deadline: 1732648566672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:06,675 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:06,675 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:06,675 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:06,731 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=416 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/B/95498f2feaf74bfc82b8cfc30f5cae2d 2024-11-26T19:15:06,739 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/C/40b66f44c98245df80224778187337b3 is 50, key is test_row_0/C:col10/1732648505292/Put/seqid=0 2024-11-26T19:15:06,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742158_1334 (size=12301) 2024-11-26T19:15:06,801 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-26T19:15:06,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. 2024-11-26T19:15:06,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. as already flushing 2024-11-26T19:15:06,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. 2024-11-26T19:15:06,801 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:06,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:06,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:06,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-26T19:15:06,954 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-26T19:15:06,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. 2024-11-26T19:15:06,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. as already flushing 2024-11-26T19:15:06,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. 2024-11-26T19:15:06,954 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:06,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:06,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:06,983 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:06,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53568 deadline: 1732648566982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:06,983 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:06,983 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:06,983 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:07,107 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-26T19:15:07,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. 2024-11-26T19:15:07,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. as already flushing 2024-11-26T19:15:07,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. 2024-11-26T19:15:07,108 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:07,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:07,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:07,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-26T19:15:07,144 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=416 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/C/40b66f44c98245df80224778187337b3 2024-11-26T19:15:07,148 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/A/2fb434edd91440b2a87edb4496632288 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/A/2fb434edd91440b2a87edb4496632288 2024-11-26T19:15:07,152 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/A/2fb434edd91440b2a87edb4496632288, entries=150, sequenceid=416, filesize=12.0 K 2024-11-26T19:15:07,153 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/B/95498f2feaf74bfc82b8cfc30f5cae2d as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/B/95498f2feaf74bfc82b8cfc30f5cae2d 2024-11-26T19:15:07,156 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/B/95498f2feaf74bfc82b8cfc30f5cae2d, entries=150, sequenceid=416, filesize=12.0 K 2024-11-26T19:15:07,157 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/C/40b66f44c98245df80224778187337b3 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/C/40b66f44c98245df80224778187337b3 2024-11-26T19:15:07,161 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/C/40b66f44c98245df80224778187337b3, entries=150, sequenceid=416, filesize=12.0 K 2024-11-26T19:15:07,161 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for 9fea48dabba44dd37de365711fba859f in 855ms, sequenceid=416, compaction requested=false 2024-11-26T19:15:07,162 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9fea48dabba44dd37de365711fba859f: 2024-11-26T19:15:07,260 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-26T19:15:07,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. 2024-11-26T19:15:07,261 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing 9fea48dabba44dd37de365711fba859f 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-26T19:15:07,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9fea48dabba44dd37de365711fba859f, store=A 2024-11-26T19:15:07,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:15:07,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9fea48dabba44dd37de365711fba859f, store=B 2024-11-26T19:15:07,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:15:07,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9fea48dabba44dd37de365711fba859f, store=C 2024-11-26T19:15:07,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:15:07,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/A/1868e5e9a1f74cb1a65efd96cf054e18 is 50, key is test_row_0/A:col10/1732648506345/Put/seqid=0 2024-11-26T19:15:07,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742159_1335 (size=12301) 2024-11-26T19:15:07,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(8855): Flush requested on 9fea48dabba44dd37de365711fba859f 2024-11-26T19:15:07,455 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. as already flushing 2024-11-26T19:15:07,504 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:07,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53556 deadline: 1732648567500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:07,505 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:07,505 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:07,505 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:07,507 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732648507490, max timestamp=1732648507503Num uniques 859; 2024-11-26T19:15:07,507 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-26T19:15:07,507 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732648507490, max timestamp=1732648507503Num uniques 859; 2024-11-26T19:15:07,507 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732648507490, max timestamp=1732648507503Num uniques 859; 2024-11-26T19:15:07,507 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-26T19:15:07,507 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-26T19:15:07,567 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:07,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53632 deadline: 1732648567561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:07,567 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:07,567 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:07,567 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:07,567 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:07,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53568 deadline: 1732648567562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:07,568 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:07,568 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:07,568 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:07,616 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:07,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53556 deadline: 1732648567612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:07,616 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:07,617 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:07,617 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:07,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-26T19:15:07,674 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:07,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53632 deadline: 1732648567671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:07,674 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:07,674 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:07,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53568 deadline: 1732648567672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:07,675 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:07,675 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:07,675 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:07,675 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:07,675 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:07,679 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=430 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/A/1868e5e9a1f74cb1a65efd96cf054e18 2024-11-26T19:15:07,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/B/39846c1d6de7483f87f7eb35b5e9a4c7 is 50, key is test_row_0/B:col10/1732648506345/Put/seqid=0 2024-11-26T19:15:07,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742160_1336 (size=12301) 2024-11-26T19:15:07,828 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:07,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53556 deadline: 1732648567822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:07,828 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:07,828 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:07,828 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:07,883 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:07,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53632 deadline: 1732648567882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:07,884 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:07,884 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:07,884 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:07,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53568 deadline: 1732648567882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:07,884 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:07,884 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:07,884 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:07,885 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:08,098 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=430 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/B/39846c1d6de7483f87f7eb35b5e9a4c7 2024-11-26T19:15:08,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/C/57ebffa52d4d446d9afcf8ff7bb4338d is 50, key is test_row_0/C:col10/1732648506345/Put/seqid=0 2024-11-26T19:15:08,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742161_1337 (size=12301) 2024-11-26T19:15:08,136 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:08,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53556 deadline: 1732648568131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:08,136 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:08,136 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:08,136 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:08,196 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:08,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53632 deadline: 1732648568192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:08,196 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:08,196 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:08,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53568 deadline: 1732648568192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:08,196 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:08,196 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:08,197 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:08,197 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:08,197 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:08,513 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=430 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/C/57ebffa52d4d446d9afcf8ff7bb4338d 2024-11-26T19:15:08,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/A/1868e5e9a1f74cb1a65efd96cf054e18 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/A/1868e5e9a1f74cb1a65efd96cf054e18 2024-11-26T19:15:08,522 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/A/1868e5e9a1f74cb1a65efd96cf054e18, entries=150, sequenceid=430, filesize=12.0 K 2024-11-26T19:15:08,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/B/39846c1d6de7483f87f7eb35b5e9a4c7 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/B/39846c1d6de7483f87f7eb35b5e9a4c7 2024-11-26T19:15:08,526 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/B/39846c1d6de7483f87f7eb35b5e9a4c7, entries=150, sequenceid=430, filesize=12.0 K 2024-11-26T19:15:08,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/C/57ebffa52d4d446d9afcf8ff7bb4338d as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/C/57ebffa52d4d446d9afcf8ff7bb4338d 2024-11-26T19:15:08,531 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/C/57ebffa52d4d446d9afcf8ff7bb4338d, entries=150, sequenceid=430, filesize=12.0 K 2024-11-26T19:15:08,532 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 9fea48dabba44dd37de365711fba859f in 1271ms, sequenceid=430, compaction requested=true 2024-11-26T19:15:08,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for 9fea48dabba44dd37de365711fba859f: 2024-11-26T19:15:08,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. 2024-11-26T19:15:08,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-11-26T19:15:08,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-11-26T19:15:08,535 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-26T19:15:08,535 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0380 sec 2024-11-26T19:15:08,537 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 2.0430 sec 2024-11-26T19:15:08,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-26T19:15:08,642 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T19:15:08,642 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T19:15:08,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-11-26T19:15:08,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-26T19:15:08,644 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T19:15:08,645 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T19:15:08,645 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T19:15:08,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(8855): Flush requested on 9fea48dabba44dd37de365711fba859f 2024-11-26T19:15:08,646 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9fea48dabba44dd37de365711fba859f 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-26T19:15:08,646 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9fea48dabba44dd37de365711fba859f, store=A 2024-11-26T19:15:08,646 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T19:15:08,646 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9fea48dabba44dd37de365711fba859f, store=B 2024-11-26T19:15:08,646 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T19:15:08,646 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9fea48dabba44dd37de365711fba859f, store=C 2024-11-26T19:15:08,646 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T19:15:08,651 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/A/6df50f2867894c5ca12f6ad6db71e35d is 50, key is test_row_0/A:col10/1732648507560/Put/seqid=0 2024-11-26T19:15:08,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742162_1338 (size=14741) 2024-11-26T19:15:08,687 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:08,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53558 deadline: 1732648568682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:08,688 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to fe76acf732b2,42785,1732648423970 for 'test_row_1' in 9fea48dabba44dd37de365711fba859f of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8215 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T19:15:08,689 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:08,689 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:08,689 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:08,692 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:08,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53556 deadline: 1732648568686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:08,693 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:08,693 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:08,693 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:08,705 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:08,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53568 deadline: 1732648568701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:08,706 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:08,706 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:08,706 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:08,716 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:08,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53632 deadline: 1732648568712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:08,716 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:08,716 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:08,716 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:08,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-26T19:15:08,798 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-26T19:15:08,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. 2024-11-26T19:15:08,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. as already flushing 2024-11-26T19:15:08,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. 2024-11-26T19:15:08,799 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:08,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:08,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:08,806 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:08,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53556 deadline: 1732648568802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:08,807 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:08,807 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:08,807 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:08,951 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-26T19:15:08,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. 2024-11-26T19:15:08,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. as already flushing 2024-11-26T19:15:08,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. 2024-11-26T19:15:08,952 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:08,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:08,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:08,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-26T19:15:09,012 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:09,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53556 deadline: 1732648569011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:09,013 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:09,013 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:09,013 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:09,060 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=466 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/A/6df50f2867894c5ca12f6ad6db71e35d 2024-11-26T19:15:09,068 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/B/f96022de149049c0b871697ab145e7c2 is 50, key is test_row_0/B:col10/1732648507560/Put/seqid=0 2024-11-26T19:15:09,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742163_1339 (size=12301) 2024-11-26T19:15:09,105 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-26T19:15:09,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. 2024-11-26T19:15:09,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. as already flushing 2024-11-26T19:15:09,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. 2024-11-26T19:15:09,105 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:09,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:09,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:09,258 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-26T19:15:09,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. 2024-11-26T19:15:09,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. as already flushing 2024-11-26T19:15:09,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. 2024-11-26T19:15:09,259 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:09,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:09,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:09,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-26T19:15:09,326 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:09,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53556 deadline: 1732648569322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:09,327 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:09,327 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:09,327 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:09,404 INFO [Thread-1237 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1237. 2024-11-26T19:15:09,404 DEBUG [Thread-1237 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T19:15:09,404 DEBUG [Thread-1237 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:15:09,404 DEBUG [Thread-1237 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:15:09,404 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T19:15:09,405 INFO [Thread-1235 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1235. 2024-11-26T19:15:09,405 DEBUG [Thread-1235 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T19:15:09,405 DEBUG [Thread-1235 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:15:09,405 DEBUG [Thread-1235 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:15:09,405 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T19:15:09,407 INFO [Thread-1233 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1233. 2024-11-26T19:15:09,407 DEBUG [Thread-1233 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T19:15:09,407 DEBUG [Thread-1233 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:15:09,407 DEBUG [Thread-1233 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:15:09,407 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T19:15:09,409 INFO [Thread-1231 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1231. 2024-11-26T19:15:09,409 DEBUG [Thread-1231 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T19:15:09,409 DEBUG [Thread-1231 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:15:09,409 DEBUG [Thread-1231 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:15:09,409 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T19:15:09,410 INFO [Thread-1229 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1229. 2024-11-26T19:15:09,410 DEBUG [Thread-1229 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T19:15:09,410 DEBUG [Thread-1229 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:15:09,410 DEBUG [Thread-1229 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:15:09,410 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T19:15:09,411 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-26T19:15:09,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. 2024-11-26T19:15:09,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. as already flushing 2024-11-26T19:15:09,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. 2024-11-26T19:15:09,411 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:09,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:09,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:09,472 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=466 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/B/f96022de149049c0b871697ab145e7c2 2024-11-26T19:15:09,479 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/C/8fe0e6f0ea38422c88a870939d136ad9 is 50, key is test_row_0/C:col10/1732648507560/Put/seqid=0 2024-11-26T19:15:09,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742164_1340 (size=12301) 2024-11-26T19:15:09,564 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-26T19:15:09,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. 2024-11-26T19:15:09,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. as already flushing 2024-11-26T19:15:09,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. 2024-11-26T19:15:09,564 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:09,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:09,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:09,712 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:09,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53568 deadline: 1732648569712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:09,712 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to fe76acf732b2,42785,1732648423970 for 'test_row_1' in 9fea48dabba44dd37de365711fba859f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2150 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T19:15:09,713 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:09,713 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:09,713 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:09,717 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-26T19:15:09,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. 2024-11-26T19:15:09,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. as already flushing 2024-11-26T19:15:09,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. 2024-11-26T19:15:09,717 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:09,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:09,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:09,732 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:09,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53632 deadline: 1732648569732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:09,733 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to fe76acf732b2,42785,1732648423970 for 'test_row_0' in 9fea48dabba44dd37de365711fba859f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2172 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T19:15:09,733 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:09,733 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:09,733 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:09,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-26T19:15:09,832 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:09,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53556 deadline: 1732648569832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:09,833 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:09,833 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:09,833 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:09,869 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-26T19:15:09,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. 2024-11-26T19:15:09,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. as already flushing 2024-11-26T19:15:09,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. 2024-11-26T19:15:09,870 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:09,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:09,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:09,882 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=466 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/C/8fe0e6f0ea38422c88a870939d136ad9 2024-11-26T19:15:09,886 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/A/6df50f2867894c5ca12f6ad6db71e35d as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/A/6df50f2867894c5ca12f6ad6db71e35d 2024-11-26T19:15:09,890 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/A/6df50f2867894c5ca12f6ad6db71e35d, entries=200, sequenceid=466, filesize=14.4 K 2024-11-26T19:15:09,890 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/B/f96022de149049c0b871697ab145e7c2 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/B/f96022de149049c0b871697ab145e7c2 2024-11-26T19:15:09,893 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/B/f96022de149049c0b871697ab145e7c2, entries=150, sequenceid=466, filesize=12.0 K 2024-11-26T19:15:09,894 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/C/8fe0e6f0ea38422c88a870939d136ad9 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/C/8fe0e6f0ea38422c88a870939d136ad9 2024-11-26T19:15:09,896 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/C/8fe0e6f0ea38422c88a870939d136ad9, entries=150, sequenceid=466, filesize=12.0 K 2024-11-26T19:15:09,897 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 9fea48dabba44dd37de365711fba859f in 1252ms, sequenceid=466, compaction requested=true 2024-11-26T19:15:09,897 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9fea48dabba44dd37de365711fba859f: 2024-11-26T19:15:09,897 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9fea48dabba44dd37de365711fba859f:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T19:15:09,897 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:15:09,897 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9fea48dabba44dd37de365711fba859f:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T19:15:09,897 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T19:15:09,897 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:15:09,897 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9fea48dabba44dd37de365711fba859f:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T19:15:09,897 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T19:15:09,897 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T19:15:09,898 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49920 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T19:15:09,898 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1541): 9fea48dabba44dd37de365711fba859f/B is initiating minor compaction (all files) 2024-11-26T19:15:09,899 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52360 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T19:15:09,899 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9fea48dabba44dd37de365711fba859f/B in TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. 2024-11-26T19:15:09,899 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1541): 9fea48dabba44dd37de365711fba859f/A is initiating minor compaction (all files) 2024-11-26T19:15:09,899 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9fea48dabba44dd37de365711fba859f/A in TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. 2024-11-26T19:15:09,899 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/B/bd5a990699814b709ea2a6953e0142e2, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/B/95498f2feaf74bfc82b8cfc30f5cae2d, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/B/39846c1d6de7483f87f7eb35b5e9a4c7, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/B/f96022de149049c0b871697ab145e7c2] into tmpdir=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp, totalSize=48.8 K 2024-11-26T19:15:09,899 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/A/01b385969a024104943b10b35fcbfa9d, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/A/2fb434edd91440b2a87edb4496632288, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/A/1868e5e9a1f74cb1a65efd96cf054e18, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/A/6df50f2867894c5ca12f6ad6db71e35d] into tmpdir=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp, totalSize=51.1 K 2024-11-26T19:15:09,899 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting 01b385969a024104943b10b35fcbfa9d, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1732648504069 2024-11-26T19:15:09,899 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting bd5a990699814b709ea2a6953e0142e2, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1732648504069 2024-11-26T19:15:09,899 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting 95498f2feaf74bfc82b8cfc30f5cae2d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=416, earliestPutTs=1732648505292 2024-11-26T19:15:09,899 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2fb434edd91440b2a87edb4496632288, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=416, earliestPutTs=1732648505292 2024-11-26T19:15:09,899 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting 39846c1d6de7483f87f7eb35b5e9a4c7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=430, earliestPutTs=1732648506310 2024-11-26T19:15:09,899 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1868e5e9a1f74cb1a65efd96cf054e18, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=430, earliestPutTs=1732648506310 2024-11-26T19:15:09,900 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting f96022de149049c0b871697ab145e7c2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=466, earliestPutTs=1732648507542 2024-11-26T19:15:09,900 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6df50f2867894c5ca12f6ad6db71e35d, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=466, earliestPutTs=1732648507542 2024-11-26T19:15:09,907 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9fea48dabba44dd37de365711fba859f#A#compaction#290 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T19:15:09,907 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9fea48dabba44dd37de365711fba859f#B#compaction#291 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T19:15:09,907 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/A/9bbf66035c29439a8113f10b490cf7bf is 50, key is test_row_0/A:col10/1732648507560/Put/seqid=0 2024-11-26T19:15:09,907 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/B/2d933453d1aa4ffc8d476b2560eedf17 is 50, key is test_row_0/B:col10/1732648507560/Put/seqid=0 2024-11-26T19:15:09,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742165_1341 (size=13153) 2024-11-26T19:15:09,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742166_1342 (size=13153) 2024-11-26T19:15:10,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(8855): Flush requested on 9fea48dabba44dd37de365711fba859f 2024-11-26T19:15:10,013 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9fea48dabba44dd37de365711fba859f 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-26T19:15:10,013 INFO [Thread-1222 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1222. 2024-11-26T19:15:10,013 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9fea48dabba44dd37de365711fba859f, store=A 2024-11-26T19:15:10,013 DEBUG [Thread-1222 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T19:15:10,013 DEBUG [Thread-1222 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:15:10,013 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:15:10,013 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9fea48dabba44dd37de365711fba859f, store=B 2024-11-26T19:15:10,013 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:15:10,013 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9fea48dabba44dd37de365711fba859f, store=C 2024-11-26T19:15:10,013 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:15:10,013 DEBUG [Thread-1222 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:15:10,013 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T19:15:10,017 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/A/a3660480c2714a6fb21c3b55f2eb08f5 is 50, key is test_row_0/A:col10/1732648508672/Put/seqid=0 2024-11-26T19:15:10,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742167_1343 (size=12301) 2024-11-26T19:15:10,022 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-26T19:15:10,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. 2024-11-26T19:15:10,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. as already flushing 2024-11-26T19:15:10,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. 2024-11-26T19:15:10,022 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:10,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:10,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:10,175 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-26T19:15:10,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. 2024-11-26T19:15:10,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. as already flushing 2024-11-26T19:15:10,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. 2024-11-26T19:15:10,175 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:10,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:10,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:10,318 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/B/2d933453d1aa4ffc8d476b2560eedf17 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/B/2d933453d1aa4ffc8d476b2560eedf17 2024-11-26T19:15:10,318 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/A/9bbf66035c29439a8113f10b490cf7bf as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/A/9bbf66035c29439a8113f10b490cf7bf 2024-11-26T19:15:10,325 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9fea48dabba44dd37de365711fba859f/A of 9fea48dabba44dd37de365711fba859f into 9bbf66035c29439a8113f10b490cf7bf(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T19:15:10,325 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9fea48dabba44dd37de365711fba859f: 2024-11-26T19:15:10,325 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9fea48dabba44dd37de365711fba859f/B of 9fea48dabba44dd37de365711fba859f into 2d933453d1aa4ffc8d476b2560eedf17(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T19:15:10,325 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., storeName=9fea48dabba44dd37de365711fba859f/A, priority=12, startTime=1732648509897; duration=0sec 2024-11-26T19:15:10,325 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9fea48dabba44dd37de365711fba859f: 2024-11-26T19:15:10,325 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., storeName=9fea48dabba44dd37de365711fba859f/B, priority=12, startTime=1732648509897; duration=0sec 2024-11-26T19:15:10,325 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T19:15:10,325 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T19:15:10,325 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9fea48dabba44dd37de365711fba859f:B 2024-11-26T19:15:10,325 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9fea48dabba44dd37de365711fba859f:A 2024-11-26T19:15:10,325 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T19:15:10,327 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49920 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T19:15:10,328 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1541): 9fea48dabba44dd37de365711fba859f/C is initiating minor compaction (all files) 2024-11-26T19:15:10,328 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9fea48dabba44dd37de365711fba859f/C in TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. 2024-11-26T19:15:10,328 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-26T19:15:10,328 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/C/95a8a434e53e48d0b5b2f0c54c3e114d, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/C/40b66f44c98245df80224778187337b3, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/C/57ebffa52d4d446d9afcf8ff7bb4338d, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/C/8fe0e6f0ea38422c88a870939d136ad9] into tmpdir=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp, totalSize=48.8 K 2024-11-26T19:15:10,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. 2024-11-26T19:15:10,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. as already flushing 2024-11-26T19:15:10,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. 2024-11-26T19:15:10,329 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:10,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:10,329 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting 95a8a434e53e48d0b5b2f0c54c3e114d, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1732648504069 2024-11-26T19:15:10,330 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting 40b66f44c98245df80224778187337b3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=416, earliestPutTs=1732648505292 2024-11-26T19:15:10,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:10,330 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting 57ebffa52d4d446d9afcf8ff7bb4338d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=430, earliestPutTs=1732648506310 2024-11-26T19:15:10,331 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting 8fe0e6f0ea38422c88a870939d136ad9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=466, earliestPutTs=1732648507542 2024-11-26T19:15:10,346 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9fea48dabba44dd37de365711fba859f#C#compaction#293 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T19:15:10,347 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/C/3254c81155ff4f37b471064e4870c582 is 50, key is test_row_0/C:col10/1732648507560/Put/seqid=0 2024-11-26T19:15:10,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742168_1344 (size=13153) 2024-11-26T19:15:10,366 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/C/3254c81155ff4f37b471064e4870c582 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/C/3254c81155ff4f37b471064e4870c582 2024-11-26T19:15:10,372 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9fea48dabba44dd37de365711fba859f/C of 9fea48dabba44dd37de365711fba859f into 3254c81155ff4f37b471064e4870c582(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T19:15:10,372 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9fea48dabba44dd37de365711fba859f: 2024-11-26T19:15:10,372 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., storeName=9fea48dabba44dd37de365711fba859f/C, priority=12, startTime=1732648509897; duration=0sec 2024-11-26T19:15:10,372 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:15:10,372 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9fea48dabba44dd37de365711fba859f:C 2024-11-26T19:15:10,421 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=477 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/A/a3660480c2714a6fb21c3b55f2eb08f5 2024-11-26T19:15:10,429 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/B/cf483482c5014444bd48d3f5ca2c07ec is 50, key is test_row_0/B:col10/1732648508672/Put/seqid=0 2024-11-26T19:15:10,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742169_1345 (size=12301) 2024-11-26T19:15:10,482 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-26T19:15:10,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. 2024-11-26T19:15:10,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. as already flushing 2024-11-26T19:15:10,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. 2024-11-26T19:15:10,482 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:10,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:10,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:10,635 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-26T19:15:10,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. 2024-11-26T19:15:10,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. as already flushing 2024-11-26T19:15:10,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. 2024-11-26T19:15:10,635 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:10,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:10,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:10,788 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-26T19:15:10,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. 2024-11-26T19:15:10,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. as already flushing 2024-11-26T19:15:10,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. 2024-11-26T19:15:10,788 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:10,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:10,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:10,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-26T19:15:10,839 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=477 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/B/cf483482c5014444bd48d3f5ca2c07ec 2024-11-26T19:15:10,844 INFO [Thread-1224 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1224. 2024-11-26T19:15:10,844 DEBUG [Thread-1224 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T19:15:10,844 DEBUG [Thread-1224 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:15:10,844 DEBUG [Thread-1224 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:15:10,844 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T19:15:10,845 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/C/9e178d7e4e09499bb6d37c255c5429a3 is 50, key is test_row_0/C:col10/1732648508672/Put/seqid=0 2024-11-26T19:15:10,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742170_1346 (size=12301) 2024-11-26T19:15:10,941 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-26T19:15:10,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. 2024-11-26T19:15:10,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. as already flushing 2024-11-26T19:15:10,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. 2024-11-26T19:15:10,941 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:10,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:10,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:11,094 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-26T19:15:11,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. 2024-11-26T19:15:11,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. as already flushing 2024-11-26T19:15:11,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. 2024-11-26T19:15:11,094 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:11,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:11,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:11,247 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-26T19:15:11,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. 2024-11-26T19:15:11,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. as already flushing 2024-11-26T19:15:11,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. 2024-11-26T19:15:11,247 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:11,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:11,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:11,249 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=477 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/C/9e178d7e4e09499bb6d37c255c5429a3 2024-11-26T19:15:11,253 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/A/a3660480c2714a6fb21c3b55f2eb08f5 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/A/a3660480c2714a6fb21c3b55f2eb08f5 2024-11-26T19:15:11,256 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/A/a3660480c2714a6fb21c3b55f2eb08f5, entries=150, sequenceid=477, filesize=12.0 K 2024-11-26T19:15:11,257 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/B/cf483482c5014444bd48d3f5ca2c07ec as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/B/cf483482c5014444bd48d3f5ca2c07ec 2024-11-26T19:15:11,260 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/B/cf483482c5014444bd48d3f5ca2c07ec, entries=150, sequenceid=477, filesize=12.0 K 2024-11-26T19:15:11,260 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/C/9e178d7e4e09499bb6d37c255c5429a3 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/C/9e178d7e4e09499bb6d37c255c5429a3 2024-11-26T19:15:11,263 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/C/9e178d7e4e09499bb6d37c255c5429a3, entries=150, sequenceid=477, filesize=12.0 K 2024-11-26T19:15:11,264 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=6.71 KB/6870 for 9fea48dabba44dd37de365711fba859f in 1250ms, sequenceid=477, compaction requested=false 2024-11-26T19:15:11,264 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9fea48dabba44dd37de365711fba859f: 2024-11-26T19:15:11,400 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-26T19:15:11,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. 2024-11-26T19:15:11,400 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing 9fea48dabba44dd37de365711fba859f 3/3 column families, dataSize=6.71 KB heapSize=20.67 KB 2024-11-26T19:15:11,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9fea48dabba44dd37de365711fba859f, store=A 2024-11-26T19:15:11,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:15:11,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9fea48dabba44dd37de365711fba859f, store=B 2024-11-26T19:15:11,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:15:11,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9fea48dabba44dd37de365711fba859f, store=C 2024-11-26T19:15:11,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:15:11,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/A/fec2f8c39ad340a1ba990dc83b5c1e32 is 50, key is test_row_2/A:col10/1732648510842/Put/seqid=0 2024-11-26T19:15:11,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742171_1347 (size=7415) 2024-11-26T19:15:11,723 INFO [Thread-1218 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1218. 2024-11-26T19:15:11,723 DEBUG [Thread-1218 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T19:15:11,723 DEBUG [Thread-1218 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:15:11,723 DEBUG [Thread-1218 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:15:11,723 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T19:15:11,753 INFO [Thread-1220 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1220. 2024-11-26T19:15:11,753 DEBUG [Thread-1220 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T19:15:11,753 DEBUG [Thread-1220 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:15:11,753 DEBUG [Thread-1220 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:15:11,753 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T19:15:11,808 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=484 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/A/fec2f8c39ad340a1ba990dc83b5c1e32 2024-11-26T19:15:11,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/B/88f51d849b9049809699fde1c64d8ecf is 50, key is test_row_2/B:col10/1732648510842/Put/seqid=0 2024-11-26T19:15:11,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742172_1348 (size=7415) 2024-11-26T19:15:12,135 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-26T19:15:12,217 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=484 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/B/88f51d849b9049809699fde1c64d8ecf 2024-11-26T19:15:12,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/C/03332432a88845449fa16e1dd297cea9 is 50, key is test_row_2/C:col10/1732648510842/Put/seqid=0 2024-11-26T19:15:12,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742173_1349 (size=7415) 2024-11-26T19:15:12,627 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=484 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/C/03332432a88845449fa16e1dd297cea9 2024-11-26T19:15:12,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/A/fec2f8c39ad340a1ba990dc83b5c1e32 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/A/fec2f8c39ad340a1ba990dc83b5c1e32 2024-11-26T19:15:12,634 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/A/fec2f8c39ad340a1ba990dc83b5c1e32, entries=50, sequenceid=484, filesize=7.2 K 2024-11-26T19:15:12,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/B/88f51d849b9049809699fde1c64d8ecf as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/B/88f51d849b9049809699fde1c64d8ecf 2024-11-26T19:15:12,638 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/B/88f51d849b9049809699fde1c64d8ecf, entries=50, sequenceid=484, filesize=7.2 K 2024-11-26T19:15:12,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/C/03332432a88845449fa16e1dd297cea9 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/C/03332432a88845449fa16e1dd297cea9 2024-11-26T19:15:12,642 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/C/03332432a88845449fa16e1dd297cea9, entries=50, sequenceid=484, filesize=7.2 K 2024-11-26T19:15:12,642 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~6.71 KB/6870, heapSize ~20.63 KB/21120, currentSize=13.42 KB/13740 for 9fea48dabba44dd37de365711fba859f in 1242ms, sequenceid=484, compaction requested=true 2024-11-26T19:15:12,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for 9fea48dabba44dd37de365711fba859f: 2024-11-26T19:15:12,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. 2024-11-26T19:15:12,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-11-26T19:15:12,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-11-26T19:15:12,645 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-26T19:15:12,645 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.9990 sec 2024-11-26T19:15:12,646 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 4.0030 sec 2024-11-26T19:15:12,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-26T19:15:12,802 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T19:15:18,724 INFO [Thread-1226 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1226. 2024-11-26T19:15:18,724 DEBUG [Thread-1226 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T19:15:18,724 DEBUG [Thread-1226 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:15:18,724 DEBUG [Thread-1226 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:15:18,724 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T19:15:18,725 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-26T19:15:18,725 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 66 2024-11-26T19:15:18,725 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 132 2024-11-26T19:15:18,725 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 16 2024-11-26T19:15:18,725 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 129 2024-11-26T19:15:18,725 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 59 2024-11-26T19:15:18,725 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-26T19:15:18,725 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2646 2024-11-26T19:15:18,725 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2566 2024-11-26T19:15:18,725 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2582 2024-11-26T19:15:18,725 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2638 2024-11-26T19:15:18,725 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2560 2024-11-26T19:15:18,725 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-26T19:15:18,725 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-26T19:15:18,725 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-26T19:15:18,725 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:15:18,725 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:15:18,726 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T19:15:18,726 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-26T19:15:18,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-26T19:15:18,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-26T19:15:18,731 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732648518731"}]},"ts":"1732648518731"} 2024-11-26T19:15:18,733 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-26T19:15:18,733 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-26T19:15:18,733 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-26T19:15:18,734 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=80, ppid=79, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9fea48dabba44dd37de365711fba859f, UNASSIGN}] 2024-11-26T19:15:18,735 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=80, ppid=79, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9fea48dabba44dd37de365711fba859f, UNASSIGN 2024-11-26T19:15:18,736 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=80 updating hbase:meta row=9fea48dabba44dd37de365711fba859f, regionState=CLOSING, regionLocation=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:18,737 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=80, ppid=79, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=9fea48dabba44dd37de365711fba859f, UNASSIGN because future has completed 2024-11-26T19:15:18,737 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-26T19:15:18,738 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; CloseRegionProcedure 9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970}] 2024-11-26T19:15:18,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-26T19:15:18,890 INFO [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] handler.UnassignRegionHandler(122): Close 9fea48dabba44dd37de365711fba859f 2024-11-26T19:15:18,890 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-26T19:15:18,890 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(1722): Closing 9fea48dabba44dd37de365711fba859f, disabling compactions & flushes 2024-11-26T19:15:18,890 INFO [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. 2024-11-26T19:15:18,890 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. 2024-11-26T19:15:18,890 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. after waiting 0 ms 2024-11-26T19:15:18,890 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. 2024-11-26T19:15:18,890 INFO [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(2902): Flushing 9fea48dabba44dd37de365711fba859f 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-11-26T19:15:18,891 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9fea48dabba44dd37de365711fba859f, store=A 2024-11-26T19:15:18,891 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:15:18,891 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9fea48dabba44dd37de365711fba859f, store=B 2024-11-26T19:15:18,891 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:15:18,891 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9fea48dabba44dd37de365711fba859f, store=C 2024-11-26T19:15:18,891 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:15:18,896 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/A/380aa89326524eef8850bc1761c12ee2 is 50, key is test_row_0/A:col10/1732648511752/Put/seqid=0 2024-11-26T19:15:18,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742174_1350 (size=9857) 2024-11-26T19:15:18,936 INFO [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=490 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/A/380aa89326524eef8850bc1761c12ee2 2024-11-26T19:15:18,950 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/B/9d5df6dd2e0840ba8db1cc6d5f43eaf6 is 50, key is test_row_0/B:col10/1732648511752/Put/seqid=0 2024-11-26T19:15:18,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742175_1351 (size=9857) 2024-11-26T19:15:18,984 INFO [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=490 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/B/9d5df6dd2e0840ba8db1cc6d5f43eaf6 2024-11-26T19:15:18,992 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/C/ef495d5f121c4ccf90a59c2f03c3c31f is 50, key is test_row_0/C:col10/1732648511752/Put/seqid=0 2024-11-26T19:15:19,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742176_1352 (size=9857) 2024-11-26T19:15:19,003 INFO [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=490 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/C/ef495d5f121c4ccf90a59c2f03c3c31f 2024-11-26T19:15:19,008 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/A/380aa89326524eef8850bc1761c12ee2 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/A/380aa89326524eef8850bc1761c12ee2 2024-11-26T19:15:19,019 INFO [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/A/380aa89326524eef8850bc1761c12ee2, entries=100, sequenceid=490, filesize=9.6 K 2024-11-26T19:15:19,020 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/B/9d5df6dd2e0840ba8db1cc6d5f43eaf6 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/B/9d5df6dd2e0840ba8db1cc6d5f43eaf6 2024-11-26T19:15:19,026 INFO [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/B/9d5df6dd2e0840ba8db1cc6d5f43eaf6, entries=100, sequenceid=490, filesize=9.6 K 2024-11-26T19:15:19,027 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/.tmp/C/ef495d5f121c4ccf90a59c2f03c3c31f as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/C/ef495d5f121c4ccf90a59c2f03c3c31f 2024-11-26T19:15:19,032 INFO [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/C/ef495d5f121c4ccf90a59c2f03c3c31f, entries=100, sequenceid=490, filesize=9.6 K 2024-11-26T19:15:19,033 INFO [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 9fea48dabba44dd37de365711fba859f in 143ms, sequenceid=490, compaction requested=true 2024-11-26T19:15:19,037 DEBUG [StoreCloser-TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/A/80ca1aabf1a841c9aadcdd990dd4c53f, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/A/833460c1b0214807a39188979120517b, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/A/25eaa41f1344439cbb967f83f03479a3, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/A/163d814450b14f6989b82eea513c776c, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/A/3c261248e40544d09f49478051ef536a, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/A/93a725690e5b40fea64ccb781269788f, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/A/1be93268fce5493eb086274e136292f2, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/A/7323d6afe09546deb22c22e22bd77e61, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/A/057584545c75402c8894f48bb1c4f5ee, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/A/94c7589f7a504502aa7b48d516454d01, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/A/f15b8d4d44c14dd990ef71f87db6bc4c, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/A/d684c2e66f464eabb838c5dbadc5bbb1, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/A/ed4375d1e1594ead8299fb77e3ea7be3, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/A/ef8afb7318294433b9f12792fa861007, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/A/26e896718c38454f8cac0d7dc59f5bd9, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/A/4e8ca4cc827749c7b3aceb2286ffcfe7, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/A/e0e1ec2b7a014376bf4101ad46c745ba, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/A/a1955113f8284fe1a4c3ee5461f76cf7, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/A/aa4d10218d08424c9fe8bd9037543bd1, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/A/6e6c0ae7f2674e1f92544ced5ab45597, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/A/01b385969a024104943b10b35fcbfa9d, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/A/d268b2561d2d46bf955f8f2c8c5b3154, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/A/2fb434edd91440b2a87edb4496632288, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/A/1868e5e9a1f74cb1a65efd96cf054e18, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/A/6df50f2867894c5ca12f6ad6db71e35d] to archive 2024-11-26T19:15:19,038 DEBUG [StoreCloser-TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-26T19:15:19,041 DEBUG [StoreCloser-TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/A/80ca1aabf1a841c9aadcdd990dd4c53f to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/A/80ca1aabf1a841c9aadcdd990dd4c53f 2024-11-26T19:15:19,043 DEBUG [StoreCloser-TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/A/833460c1b0214807a39188979120517b to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/A/833460c1b0214807a39188979120517b 2024-11-26T19:15:19,044 DEBUG [StoreCloser-TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/A/25eaa41f1344439cbb967f83f03479a3 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/A/25eaa41f1344439cbb967f83f03479a3 2024-11-26T19:15:19,046 DEBUG [StoreCloser-TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/A/163d814450b14f6989b82eea513c776c to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/A/163d814450b14f6989b82eea513c776c 2024-11-26T19:15:19,047 DEBUG [StoreCloser-TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/A/3c261248e40544d09f49478051ef536a to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/A/3c261248e40544d09f49478051ef536a 2024-11-26T19:15:19,049 DEBUG [StoreCloser-TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/A/93a725690e5b40fea64ccb781269788f to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/A/93a725690e5b40fea64ccb781269788f 2024-11-26T19:15:19,050 DEBUG [StoreCloser-TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/A/1be93268fce5493eb086274e136292f2 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/A/1be93268fce5493eb086274e136292f2 2024-11-26T19:15:19,051 DEBUG [StoreCloser-TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/A/7323d6afe09546deb22c22e22bd77e61 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/A/7323d6afe09546deb22c22e22bd77e61 2024-11-26T19:15:19,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-26T19:15:19,053 DEBUG [StoreCloser-TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/A/057584545c75402c8894f48bb1c4f5ee to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/A/057584545c75402c8894f48bb1c4f5ee 2024-11-26T19:15:19,054 DEBUG [StoreCloser-TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/A/94c7589f7a504502aa7b48d516454d01 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/A/94c7589f7a504502aa7b48d516454d01 2024-11-26T19:15:19,055 DEBUG [StoreCloser-TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/A/f15b8d4d44c14dd990ef71f87db6bc4c to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/A/f15b8d4d44c14dd990ef71f87db6bc4c 2024-11-26T19:15:19,056 DEBUG [StoreCloser-TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/A/d684c2e66f464eabb838c5dbadc5bbb1 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/A/d684c2e66f464eabb838c5dbadc5bbb1 2024-11-26T19:15:19,057 DEBUG [StoreCloser-TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/A/ed4375d1e1594ead8299fb77e3ea7be3 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/A/ed4375d1e1594ead8299fb77e3ea7be3 2024-11-26T19:15:19,058 DEBUG [StoreCloser-TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/A/ef8afb7318294433b9f12792fa861007 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/A/ef8afb7318294433b9f12792fa861007 2024-11-26T19:15:19,060 DEBUG [StoreCloser-TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/A/26e896718c38454f8cac0d7dc59f5bd9 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/A/26e896718c38454f8cac0d7dc59f5bd9 2024-11-26T19:15:19,065 DEBUG [StoreCloser-TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/A/4e8ca4cc827749c7b3aceb2286ffcfe7 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/A/4e8ca4cc827749c7b3aceb2286ffcfe7 2024-11-26T19:15:19,066 DEBUG [StoreCloser-TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/A/e0e1ec2b7a014376bf4101ad46c745ba to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/A/e0e1ec2b7a014376bf4101ad46c745ba 2024-11-26T19:15:19,067 DEBUG [StoreCloser-TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/A/a1955113f8284fe1a4c3ee5461f76cf7 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/A/a1955113f8284fe1a4c3ee5461f76cf7 2024-11-26T19:15:19,068 DEBUG [StoreCloser-TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/A/aa4d10218d08424c9fe8bd9037543bd1 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/A/aa4d10218d08424c9fe8bd9037543bd1 2024-11-26T19:15:19,069 DEBUG [StoreCloser-TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/A/6e6c0ae7f2674e1f92544ced5ab45597 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/A/6e6c0ae7f2674e1f92544ced5ab45597 2024-11-26T19:15:19,070 DEBUG [StoreCloser-TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/A/01b385969a024104943b10b35fcbfa9d to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/A/01b385969a024104943b10b35fcbfa9d 2024-11-26T19:15:19,071 DEBUG [StoreCloser-TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/A/d268b2561d2d46bf955f8f2c8c5b3154 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/A/d268b2561d2d46bf955f8f2c8c5b3154 2024-11-26T19:15:19,072 DEBUG [StoreCloser-TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/A/2fb434edd91440b2a87edb4496632288 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/A/2fb434edd91440b2a87edb4496632288 2024-11-26T19:15:19,073 DEBUG [StoreCloser-TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/A/1868e5e9a1f74cb1a65efd96cf054e18 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/A/1868e5e9a1f74cb1a65efd96cf054e18 2024-11-26T19:15:19,074 DEBUG [StoreCloser-TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/A/6df50f2867894c5ca12f6ad6db71e35d to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/A/6df50f2867894c5ca12f6ad6db71e35d 2024-11-26T19:15:19,079 DEBUG [StoreCloser-TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/B/99554e299218496cb82c5fa1a4ef37d7, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/B/4da3c279be304242bd6b601b9f924f60, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/B/614cf91a41c444ee98514d034f217c58, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/B/d1c197dd4d7e4364a1852d5387c38afb, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/B/6ad1a36bf405489badf67747f44426a5, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/B/ac62361e7ed844ddb5b5b3fd013be089, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/B/6dce579f4c34441bbad425f55936e1a4, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/B/81747701670d4962af9d8e635fd9142a, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/B/f4779b77e81d4463904ba48cf953b2d1, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/B/afa345985f954ec09e440a65816b226d, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/B/a49a748b1b4c499397e7207d90c7a0aa, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/B/8742c686f20c472eb1cd0f7b30a9efd6, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/B/40c09f8563cf4277a86aa84f990367ae, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/B/3fd4076cfe914cddbfc11bbf47ebe16a, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/B/0f1594892dbe429b947e32e365b37685, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/B/72c0dc24c74e47ebb78105ddf4b18417, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/B/c618467f02014536825687fe8d6242ad, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/B/e673290407d2424b8f67ea4c4af8c9ab, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/B/475942e1b59f4b6ba54430773423a0cb, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/B/36a8189df54f408ba4c725468ef7f3f6, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/B/bd5a990699814b709ea2a6953e0142e2, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/B/ac28fcfe39004e69acb9e67d5431c1fd, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/B/95498f2feaf74bfc82b8cfc30f5cae2d, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/B/39846c1d6de7483f87f7eb35b5e9a4c7, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/B/f96022de149049c0b871697ab145e7c2] to archive 2024-11-26T19:15:19,080 DEBUG [StoreCloser-TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-26T19:15:19,081 DEBUG [StoreCloser-TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/B/99554e299218496cb82c5fa1a4ef37d7 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/B/99554e299218496cb82c5fa1a4ef37d7 2024-11-26T19:15:19,083 DEBUG [StoreCloser-TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/B/4da3c279be304242bd6b601b9f924f60 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/B/4da3c279be304242bd6b601b9f924f60 2024-11-26T19:15:19,084 DEBUG [StoreCloser-TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/B/614cf91a41c444ee98514d034f217c58 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/B/614cf91a41c444ee98514d034f217c58 2024-11-26T19:15:19,088 DEBUG [StoreCloser-TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/B/d1c197dd4d7e4364a1852d5387c38afb to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/B/d1c197dd4d7e4364a1852d5387c38afb 2024-11-26T19:15:19,092 DEBUG [StoreCloser-TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/B/6ad1a36bf405489badf67747f44426a5 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/B/6ad1a36bf405489badf67747f44426a5 2024-11-26T19:15:19,093 DEBUG [StoreCloser-TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/B/ac62361e7ed844ddb5b5b3fd013be089 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/B/ac62361e7ed844ddb5b5b3fd013be089 2024-11-26T19:15:19,094 DEBUG [StoreCloser-TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/B/6dce579f4c34441bbad425f55936e1a4 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/B/6dce579f4c34441bbad425f55936e1a4 2024-11-26T19:15:19,095 DEBUG [StoreCloser-TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/B/81747701670d4962af9d8e635fd9142a to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/B/81747701670d4962af9d8e635fd9142a 2024-11-26T19:15:19,096 DEBUG [StoreCloser-TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/B/f4779b77e81d4463904ba48cf953b2d1 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/B/f4779b77e81d4463904ba48cf953b2d1 2024-11-26T19:15:19,097 DEBUG [StoreCloser-TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/B/afa345985f954ec09e440a65816b226d to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/B/afa345985f954ec09e440a65816b226d 2024-11-26T19:15:19,098 DEBUG [StoreCloser-TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/B/a49a748b1b4c499397e7207d90c7a0aa to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/B/a49a748b1b4c499397e7207d90c7a0aa 2024-11-26T19:15:19,099 DEBUG [StoreCloser-TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/B/8742c686f20c472eb1cd0f7b30a9efd6 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/B/8742c686f20c472eb1cd0f7b30a9efd6 2024-11-26T19:15:19,099 DEBUG [StoreCloser-TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/B/40c09f8563cf4277a86aa84f990367ae to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/B/40c09f8563cf4277a86aa84f990367ae 2024-11-26T19:15:19,100 DEBUG [StoreCloser-TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/B/3fd4076cfe914cddbfc11bbf47ebe16a to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/B/3fd4076cfe914cddbfc11bbf47ebe16a 2024-11-26T19:15:19,101 DEBUG [StoreCloser-TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/B/0f1594892dbe429b947e32e365b37685 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/B/0f1594892dbe429b947e32e365b37685 2024-11-26T19:15:19,102 DEBUG [StoreCloser-TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/B/72c0dc24c74e47ebb78105ddf4b18417 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/B/72c0dc24c74e47ebb78105ddf4b18417 2024-11-26T19:15:19,103 DEBUG [StoreCloser-TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/B/c618467f02014536825687fe8d6242ad to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/B/c618467f02014536825687fe8d6242ad 2024-11-26T19:15:19,104 DEBUG [StoreCloser-TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/B/e673290407d2424b8f67ea4c4af8c9ab to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/B/e673290407d2424b8f67ea4c4af8c9ab 2024-11-26T19:15:19,105 DEBUG [StoreCloser-TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/B/475942e1b59f4b6ba54430773423a0cb to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/B/475942e1b59f4b6ba54430773423a0cb 2024-11-26T19:15:19,108 DEBUG [StoreCloser-TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/B/36a8189df54f408ba4c725468ef7f3f6 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/B/36a8189df54f408ba4c725468ef7f3f6 2024-11-26T19:15:19,110 DEBUG [StoreCloser-TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/B/bd5a990699814b709ea2a6953e0142e2 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/B/bd5a990699814b709ea2a6953e0142e2 2024-11-26T19:15:19,111 DEBUG [StoreCloser-TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/B/ac28fcfe39004e69acb9e67d5431c1fd to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/B/ac28fcfe39004e69acb9e67d5431c1fd 2024-11-26T19:15:19,112 DEBUG [StoreCloser-TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/B/95498f2feaf74bfc82b8cfc30f5cae2d to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/B/95498f2feaf74bfc82b8cfc30f5cae2d 2024-11-26T19:15:19,113 DEBUG [StoreCloser-TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/B/39846c1d6de7483f87f7eb35b5e9a4c7 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/B/39846c1d6de7483f87f7eb35b5e9a4c7 2024-11-26T19:15:19,116 DEBUG [StoreCloser-TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/B/f96022de149049c0b871697ab145e7c2 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/B/f96022de149049c0b871697ab145e7c2 2024-11-26T19:15:19,117 DEBUG [StoreCloser-TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/C/33185f0158b44276b42db1e10ea0d23c, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/C/346cdda36bf5484ba57db7c586fba603, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/C/2f9d41ec937741dfa5d7ae2237b7cfd2, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/C/39c215788bc843a89c221b150d0b3fe6, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/C/99c6e833533b4d67a1917fc913e5c311, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/C/830fa129d23b4d4c8e8726903e499285, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/C/e219d6fa7b454ece86951192720086e3, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/C/c85c2e9abb874214b64322183a5629b7, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/C/7d918e0ee7324aaf9702f798c320d8e2, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/C/8d65a7b76005450db2e45cf5365dcb26, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/C/d3f5af72a5084ea298a468a57385e222, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/C/06dfc70f3cde46b39562ce542b6ad0d1, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/C/870540b8ca6d4daa94b00196a17f1e7c, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/C/2219ca97b35c4801b8d2244c3a103e6b, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/C/d26df17321564eca93704812994d6503, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/C/5577393f361b45f989c7c7cb5e5e52ec, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/C/989c1bef759440419fbd0b43bed9b6cb, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/C/f972aaf11a6140dfa1bc08bdbdfbe26b, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/C/ea88edf691d54926836dbbd48ba0f773, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/C/ea2759e5f801454cb6a5f87d75121ee2, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/C/95a8a434e53e48d0b5b2f0c54c3e114d, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/C/6b282d4f0bca438189eb2b19645b1717, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/C/40b66f44c98245df80224778187337b3, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/C/57ebffa52d4d446d9afcf8ff7bb4338d, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/C/8fe0e6f0ea38422c88a870939d136ad9] to archive 2024-11-26T19:15:19,118 DEBUG [StoreCloser-TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-26T19:15:19,120 DEBUG [StoreCloser-TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/C/33185f0158b44276b42db1e10ea0d23c to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/C/33185f0158b44276b42db1e10ea0d23c 2024-11-26T19:15:19,121 DEBUG [StoreCloser-TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/C/346cdda36bf5484ba57db7c586fba603 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/C/346cdda36bf5484ba57db7c586fba603 2024-11-26T19:15:19,122 DEBUG [StoreCloser-TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/C/2f9d41ec937741dfa5d7ae2237b7cfd2 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/C/2f9d41ec937741dfa5d7ae2237b7cfd2 2024-11-26T19:15:19,124 DEBUG [StoreCloser-TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/C/39c215788bc843a89c221b150d0b3fe6 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/C/39c215788bc843a89c221b150d0b3fe6 2024-11-26T19:15:19,125 DEBUG [StoreCloser-TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/C/99c6e833533b4d67a1917fc913e5c311 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/C/99c6e833533b4d67a1917fc913e5c311 2024-11-26T19:15:19,126 DEBUG [StoreCloser-TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/C/830fa129d23b4d4c8e8726903e499285 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/C/830fa129d23b4d4c8e8726903e499285 2024-11-26T19:15:19,127 DEBUG [StoreCloser-TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/C/e219d6fa7b454ece86951192720086e3 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/C/e219d6fa7b454ece86951192720086e3 2024-11-26T19:15:19,128 DEBUG [StoreCloser-TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/C/c85c2e9abb874214b64322183a5629b7 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/C/c85c2e9abb874214b64322183a5629b7 2024-11-26T19:15:19,129 DEBUG [StoreCloser-TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/C/7d918e0ee7324aaf9702f798c320d8e2 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/C/7d918e0ee7324aaf9702f798c320d8e2 2024-11-26T19:15:19,130 DEBUG [StoreCloser-TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/C/8d65a7b76005450db2e45cf5365dcb26 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/C/8d65a7b76005450db2e45cf5365dcb26 2024-11-26T19:15:19,131 DEBUG [StoreCloser-TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/C/d3f5af72a5084ea298a468a57385e222 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/C/d3f5af72a5084ea298a468a57385e222 2024-11-26T19:15:19,132 DEBUG [StoreCloser-TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/C/06dfc70f3cde46b39562ce542b6ad0d1 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/C/06dfc70f3cde46b39562ce542b6ad0d1 2024-11-26T19:15:19,133 DEBUG [StoreCloser-TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/C/870540b8ca6d4daa94b00196a17f1e7c to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/C/870540b8ca6d4daa94b00196a17f1e7c 2024-11-26T19:15:19,134 DEBUG [StoreCloser-TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/C/2219ca97b35c4801b8d2244c3a103e6b to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/C/2219ca97b35c4801b8d2244c3a103e6b 2024-11-26T19:15:19,135 DEBUG [StoreCloser-TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/C/d26df17321564eca93704812994d6503 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/C/d26df17321564eca93704812994d6503 2024-11-26T19:15:19,136 DEBUG [StoreCloser-TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/C/5577393f361b45f989c7c7cb5e5e52ec to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/C/5577393f361b45f989c7c7cb5e5e52ec 2024-11-26T19:15:19,137 DEBUG [StoreCloser-TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/C/989c1bef759440419fbd0b43bed9b6cb to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/C/989c1bef759440419fbd0b43bed9b6cb 2024-11-26T19:15:19,138 DEBUG [StoreCloser-TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/C/f972aaf11a6140dfa1bc08bdbdfbe26b to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/C/f972aaf11a6140dfa1bc08bdbdfbe26b 2024-11-26T19:15:19,139 DEBUG [StoreCloser-TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/C/ea88edf691d54926836dbbd48ba0f773 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/C/ea88edf691d54926836dbbd48ba0f773 2024-11-26T19:15:19,140 DEBUG [StoreCloser-TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/C/ea2759e5f801454cb6a5f87d75121ee2 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/C/ea2759e5f801454cb6a5f87d75121ee2 2024-11-26T19:15:19,141 DEBUG [StoreCloser-TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/C/95a8a434e53e48d0b5b2f0c54c3e114d to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/C/95a8a434e53e48d0b5b2f0c54c3e114d 2024-11-26T19:15:19,142 DEBUG [StoreCloser-TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/C/6b282d4f0bca438189eb2b19645b1717 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/C/6b282d4f0bca438189eb2b19645b1717 2024-11-26T19:15:19,142 DEBUG [StoreCloser-TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/C/40b66f44c98245df80224778187337b3 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/C/40b66f44c98245df80224778187337b3 2024-11-26T19:15:19,144 DEBUG [StoreCloser-TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/C/57ebffa52d4d446d9afcf8ff7bb4338d to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/C/57ebffa52d4d446d9afcf8ff7bb4338d 2024-11-26T19:15:19,151 DEBUG [StoreCloser-TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/C/8fe0e6f0ea38422c88a870939d136ad9 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/C/8fe0e6f0ea38422c88a870939d136ad9 2024-11-26T19:15:19,180 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/recovered.edits/493.seqid, newMaxSeqId=493, maxSeqId=1 2024-11-26T19:15:19,181 INFO [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. 2024-11-26T19:15:19,181 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(1676): Region close journal for 9fea48dabba44dd37de365711fba859f: Waiting for close lock at 1732648518890Running coprocessor pre-close hooks at 1732648518890Disabling compacts and flushes for region at 1732648518890Disabling writes for close at 1732648518890Obtaining lock to block concurrent updates at 1732648518890Preparing flush snapshotting stores in 9fea48dabba44dd37de365711fba859f at 1732648518890Finished memstore snapshotting TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=61920, getOffHeapSize=0, getCellsCount=450 at 1732648518891 (+1 ms)Flushing stores of TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f. at 1732648518892 (+1 ms)Flushing 9fea48dabba44dd37de365711fba859f/A: creating writer at 1732648518892Flushing 9fea48dabba44dd37de365711fba859f/A: appending metadata at 1732648518895 (+3 ms)Flushing 9fea48dabba44dd37de365711fba859f/A: closing flushed file at 1732648518895Flushing 9fea48dabba44dd37de365711fba859f/B: creating writer at 1732648518942 (+47 ms)Flushing 9fea48dabba44dd37de365711fba859f/B: appending metadata at 1732648518949 (+7 ms)Flushing 9fea48dabba44dd37de365711fba859f/B: closing flushed file at 1732648518949Flushing 9fea48dabba44dd37de365711fba859f/C: creating writer at 1732648518989 (+40 ms)Flushing 9fea48dabba44dd37de365711fba859f/C: appending metadata at 1732648518991 (+2 ms)Flushing 9fea48dabba44dd37de365711fba859f/C: closing flushed file at 1732648518992 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2888502b: reopening flushed file at 1732648519007 (+15 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@232887a0: reopening flushed file at 1732648519019 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7a8ef583: reopening flushed file at 1732648519026 (+7 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 9fea48dabba44dd37de365711fba859f in 143ms, sequenceid=490, compaction requested=true at 1732648519033 (+7 ms)Writing region close event to WAL at 1732648519168 (+135 ms)Running coprocessor post-close hooks at 1732648519181 (+13 ms)Closed at 1732648519181 2024-11-26T19:15:19,184 INFO [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] handler.UnassignRegionHandler(157): Closed 9fea48dabba44dd37de365711fba859f 2024-11-26T19:15:19,185 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=80 updating hbase:meta row=9fea48dabba44dd37de365711fba859f, regionState=CLOSED 2024-11-26T19:15:19,186 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=81, ppid=80, state=RUNNABLE, hasLock=false; CloseRegionProcedure 9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 because future has completed 2024-11-26T19:15:19,189 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-11-26T19:15:19,189 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; CloseRegionProcedure 9fea48dabba44dd37de365711fba859f, server=fe76acf732b2,42785,1732648423970 in 450 msec 2024-11-26T19:15:19,192 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=80, resume processing ppid=79 2024-11-26T19:15:19,192 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, ppid=79, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9fea48dabba44dd37de365711fba859f, UNASSIGN in 456 msec 2024-11-26T19:15:19,197 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732648519196"}]},"ts":"1732648519196"} 2024-11-26T19:15:19,199 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-26T19:15:19,199 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-26T19:15:19,199 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-11-26T19:15:19,201 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 473 msec 2024-11-26T19:15:19,201 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 460 msec 2024-11-26T19:15:19,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-26T19:15:19,363 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-26T19:15:19,363 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-26T19:15:19,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] procedure2.ProcedureExecutor(1139): Stored pid=82, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T19:15:19,366 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=82, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T19:15:19,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-26T19:15:19,366 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=82, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T19:15:19,369 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f 2024-11-26T19:15:19,371 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/A, FileablePath, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/B, FileablePath, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/C, FileablePath, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/recovered.edits] 2024-11-26T19:15:19,375 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/A/380aa89326524eef8850bc1761c12ee2 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/A/380aa89326524eef8850bc1761c12ee2 2024-11-26T19:15:19,381 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/A/9bbf66035c29439a8113f10b490cf7bf to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/A/9bbf66035c29439a8113f10b490cf7bf 2024-11-26T19:15:19,383 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/A/a3660480c2714a6fb21c3b55f2eb08f5 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/A/a3660480c2714a6fb21c3b55f2eb08f5 2024-11-26T19:15:19,384 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/A/fec2f8c39ad340a1ba990dc83b5c1e32 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/A/fec2f8c39ad340a1ba990dc83b5c1e32 2024-11-26T19:15:19,386 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/B/2d933453d1aa4ffc8d476b2560eedf17 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/B/2d933453d1aa4ffc8d476b2560eedf17 2024-11-26T19:15:19,388 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/B/88f51d849b9049809699fde1c64d8ecf to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/B/88f51d849b9049809699fde1c64d8ecf 2024-11-26T19:15:19,389 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/B/9d5df6dd2e0840ba8db1cc6d5f43eaf6 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/B/9d5df6dd2e0840ba8db1cc6d5f43eaf6 2024-11-26T19:15:19,391 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/B/cf483482c5014444bd48d3f5ca2c07ec to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/B/cf483482c5014444bd48d3f5ca2c07ec 2024-11-26T19:15:19,394 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/C/03332432a88845449fa16e1dd297cea9 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/C/03332432a88845449fa16e1dd297cea9 2024-11-26T19:15:19,395 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/C/3254c81155ff4f37b471064e4870c582 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/C/3254c81155ff4f37b471064e4870c582 2024-11-26T19:15:19,397 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/C/9e178d7e4e09499bb6d37c255c5429a3 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/C/9e178d7e4e09499bb6d37c255c5429a3 2024-11-26T19:15:19,399 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/C/ef495d5f121c4ccf90a59c2f03c3c31f to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/C/ef495d5f121c4ccf90a59c2f03c3c31f 2024-11-26T19:15:19,404 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/recovered.edits/493.seqid to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f/recovered.edits/493.seqid 2024-11-26T19:15:19,404 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/9fea48dabba44dd37de365711fba859f 2024-11-26T19:15:19,404 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-26T19:15:19,407 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=82, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T19:15:19,421 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-26T19:15:19,425 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-26T19:15:19,427 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=82, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T19:15:19,427 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-26T19:15:19,428 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732648519427"}]},"ts":"9223372036854775807"} 2024-11-26T19:15:19,430 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-26T19:15:19,430 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 9fea48dabba44dd37de365711fba859f, NAME => 'TestAcidGuarantees,,1732648487211.9fea48dabba44dd37de365711fba859f.', STARTKEY => '', ENDKEY => ''}] 2024-11-26T19:15:19,430 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-26T19:15:19,431 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732648519430"}]},"ts":"9223372036854775807"} 2024-11-26T19:15:19,433 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-26T19:15:19,434 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=82, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T19:15:19,435 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 71 msec 2024-11-26T19:15:19,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-26T19:15:19,472 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-26T19:15:19,472 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-26T19:15:19,487 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=247 (was 251), OpenFileDescriptor=449 (was 459), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=390 (was 483), ProcessCount=11 (was 11), AvailableMemoryMB=5866 (was 6596) 2024-11-26T19:15:19,501 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=247, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=390, ProcessCount=11, AvailableMemoryMB=5864 2024-11-26T19:15:19,503 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-26T19:15:19,503 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-26T19:15:19,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] procedure2.ProcedureExecutor(1139): Stored pid=83, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-26T19:15:19,505 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=83, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-26T19:15:19,506 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:19,506 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 83 2024-11-26T19:15:19,507 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=83, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-26T19:15:19,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=83 2024-11-26T19:15:19,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742177_1353 (size=1061) 2024-11-26T19:15:19,525 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 082e64a1d02e824a02737c5fd026f5bd, NAME => 'TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16 2024-11-26T19:15:19,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742178_1354 (size=53) 2024-11-26T19:15:19,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=83 2024-11-26T19:15:19,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=83 2024-11-26T19:15:19,935 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T19:15:19,935 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 082e64a1d02e824a02737c5fd026f5bd, disabling compactions & flushes 2024-11-26T19:15:19,935 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd. 2024-11-26T19:15:19,935 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd. 2024-11-26T19:15:19,935 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd. after waiting 0 ms 2024-11-26T19:15:19,935 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd. 2024-11-26T19:15:19,935 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd. 2024-11-26T19:15:19,935 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 082e64a1d02e824a02737c5fd026f5bd: Waiting for close lock at 1732648519935Disabling compacts and flushes for region at 1732648519935Disabling writes for close at 1732648519935Writing region close event to WAL at 1732648519935Closed at 1732648519935 2024-11-26T19:15:19,936 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=83, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-26T19:15:19,937 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732648519936"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732648519936"}]},"ts":"1732648519936"} 2024-11-26T19:15:19,938 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-26T19:15:19,939 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=83, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-26T19:15:19,939 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732648519939"}]},"ts":"1732648519939"} 2024-11-26T19:15:19,941 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-26T19:15:19,941 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=082e64a1d02e824a02737c5fd026f5bd, ASSIGN}] 2024-11-26T19:15:19,943 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=082e64a1d02e824a02737c5fd026f5bd, ASSIGN 2024-11-26T19:15:19,943 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=082e64a1d02e824a02737c5fd026f5bd, ASSIGN; state=OFFLINE, location=fe76acf732b2,42785,1732648423970; forceNewPlan=false, retain=false 2024-11-26T19:15:20,094 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=082e64a1d02e824a02737c5fd026f5bd, regionState=OPENING, regionLocation=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:20,096 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=082e64a1d02e824a02737c5fd026f5bd, ASSIGN because future has completed 2024-11-26T19:15:20,096 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE, hasLock=false; OpenRegionProcedure 082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970}] 2024-11-26T19:15:20,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=83 2024-11-26T19:15:20,250 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd. 2024-11-26T19:15:20,250 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(7752): Opening region: {ENCODED => 082e64a1d02e824a02737c5fd026f5bd, NAME => 'TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd.', STARTKEY => '', ENDKEY => ''} 2024-11-26T19:15:20,251 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 082e64a1d02e824a02737c5fd026f5bd 2024-11-26T19:15:20,251 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T19:15:20,251 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(7794): checking encryption for 082e64a1d02e824a02737c5fd026f5bd 2024-11-26T19:15:20,251 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(7797): checking classloading for 082e64a1d02e824a02737c5fd026f5bd 2024-11-26T19:15:20,252 INFO [StoreOpener-082e64a1d02e824a02737c5fd026f5bd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 082e64a1d02e824a02737c5fd026f5bd 2024-11-26T19:15:20,253 INFO [StoreOpener-082e64a1d02e824a02737c5fd026f5bd-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-26T19:15:20,253 INFO [StoreOpener-082e64a1d02e824a02737c5fd026f5bd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 082e64a1d02e824a02737c5fd026f5bd columnFamilyName A 2024-11-26T19:15:20,253 DEBUG [StoreOpener-082e64a1d02e824a02737c5fd026f5bd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:20,253 INFO [StoreOpener-082e64a1d02e824a02737c5fd026f5bd-1 {}] regionserver.HStore(327): Store=082e64a1d02e824a02737c5fd026f5bd/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T19:15:20,254 INFO [StoreOpener-082e64a1d02e824a02737c5fd026f5bd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 082e64a1d02e824a02737c5fd026f5bd 2024-11-26T19:15:20,254 INFO [StoreOpener-082e64a1d02e824a02737c5fd026f5bd-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-26T19:15:20,254 INFO [StoreOpener-082e64a1d02e824a02737c5fd026f5bd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 082e64a1d02e824a02737c5fd026f5bd columnFamilyName B 2024-11-26T19:15:20,254 DEBUG [StoreOpener-082e64a1d02e824a02737c5fd026f5bd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:20,255 INFO [StoreOpener-082e64a1d02e824a02737c5fd026f5bd-1 {}] regionserver.HStore(327): Store=082e64a1d02e824a02737c5fd026f5bd/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T19:15:20,255 INFO [StoreOpener-082e64a1d02e824a02737c5fd026f5bd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 082e64a1d02e824a02737c5fd026f5bd 2024-11-26T19:15:20,255 INFO [StoreOpener-082e64a1d02e824a02737c5fd026f5bd-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-26T19:15:20,256 INFO [StoreOpener-082e64a1d02e824a02737c5fd026f5bd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 082e64a1d02e824a02737c5fd026f5bd columnFamilyName C 2024-11-26T19:15:20,256 DEBUG [StoreOpener-082e64a1d02e824a02737c5fd026f5bd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:20,256 INFO [StoreOpener-082e64a1d02e824a02737c5fd026f5bd-1 {}] regionserver.HStore(327): Store=082e64a1d02e824a02737c5fd026f5bd/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T19:15:20,256 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd. 2024-11-26T19:15:20,256 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(1038): replaying wal for 082e64a1d02e824a02737c5fd026f5bd 2024-11-26T19:15:20,257 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd 2024-11-26T19:15:20,257 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd 2024-11-26T19:15:20,257 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(1048): stopping wal replay for 082e64a1d02e824a02737c5fd026f5bd 2024-11-26T19:15:20,257 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(1060): Cleaning up temporary data for 082e64a1d02e824a02737c5fd026f5bd 2024-11-26T19:15:20,258 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-26T19:15:20,259 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(1093): writing seq id for 082e64a1d02e824a02737c5fd026f5bd 2024-11-26T19:15:20,260 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T19:15:20,260 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(1114): Opened 082e64a1d02e824a02737c5fd026f5bd; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67847187, jitterRate=0.011001870036125183}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-26T19:15:20,260 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 082e64a1d02e824a02737c5fd026f5bd 2024-11-26T19:15:20,261 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(1006): Region open journal for 082e64a1d02e824a02737c5fd026f5bd: Running coprocessor pre-open hook at 1732648520251Writing region info on filesystem at 1732648520251Initializing all the Stores at 1732648520251Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732648520251Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732648520252 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732648520252Cleaning up temporary data from old regions at 1732648520257 (+5 ms)Running coprocessor post-open hooks at 1732648520260 (+3 ms)Region opened successfully at 1732648520261 (+1 ms) 2024-11-26T19:15:20,261 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., pid=85, masterSystemTime=1732648520247 2024-11-26T19:15:20,263 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd. 2024-11-26T19:15:20,263 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd. 2024-11-26T19:15:20,263 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=082e64a1d02e824a02737c5fd026f5bd, regionState=OPEN, openSeqNum=2, regionLocation=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:20,265 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=85, ppid=84, state=RUNNABLE, hasLock=false; OpenRegionProcedure 082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 because future has completed 2024-11-26T19:15:20,267 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-11-26T19:15:20,267 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; OpenRegionProcedure 082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 in 169 msec 2024-11-26T19:15:20,268 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=84, resume processing ppid=83 2024-11-26T19:15:20,268 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, ppid=83, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=082e64a1d02e824a02737c5fd026f5bd, ASSIGN in 326 msec 2024-11-26T19:15:20,269 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=83, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-26T19:15:20,269 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732648520269"}]},"ts":"1732648520269"} 2024-11-26T19:15:20,270 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-26T19:15:20,271 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=83, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-26T19:15:20,272 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 768 msec 2024-11-26T19:15:20,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=83 2024-11-26T19:15:20,642 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-26T19:15:20,643 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@211104f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:15:20,643 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fe76acf732b2,33811,-1 for getting cluster id 2024-11-26T19:15:20,643 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T19:15:20,644 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3cec4e24-71db-4cd0-b63a-4ea30a9b160b' 2024-11-26T19:15:20,644 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T19:15:20,644 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3cec4e24-71db-4cd0-b63a-4ea30a9b160b" 2024-11-26T19:15:20,644 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f2a80c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:15:20,644 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fe76acf732b2,33811,-1] 2024-11-26T19:15:20,645 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T19:15:20,645 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:15:20,646 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35062, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T19:15:20,646 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4699108c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:15:20,646 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T19:15:20,647 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fe76acf732b2,42785,1732648423970, seqNum=-1] 2024-11-26T19:15:20,647 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T19:15:20,648 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58838, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T19:15:20,649 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-26T19:15:20,649 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is fe76acf732b2,33811,1732648423099 2024-11-26T19:15:20,649 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@6f98b82a 2024-11-26T19:15:20,649 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-26T19:15:20,650 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35072, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-26T19:15:20,651 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-26T19:15:20,651 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-26T19:15:20,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] procedure2.ProcedureExecutor(1139): Stored pid=86, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-26T19:15:20,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742179_1355 (size=1097) 2024-11-26T19:15:21,062 DEBUG [PEWorker-2 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-26T19:15:21,062 INFO [PEWorker-2 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-26T19:15:21,064 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-26T19:15:21,066 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=082e64a1d02e824a02737c5fd026f5bd, REOPEN/MOVE}] 2024-11-26T19:15:21,067 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=082e64a1d02e824a02737c5fd026f5bd, REOPEN/MOVE 2024-11-26T19:15:21,067 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=082e64a1d02e824a02737c5fd026f5bd, regionState=CLOSING, regionLocation=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:21,068 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=082e64a1d02e824a02737c5fd026f5bd, REOPEN/MOVE because future has completed 2024-11-26T19:15:21,069 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-26T19:15:21,069 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE, hasLock=false; CloseRegionProcedure 082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970}] 2024-11-26T19:15:21,221 INFO [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] handler.UnassignRegionHandler(122): Close 082e64a1d02e824a02737c5fd026f5bd 2024-11-26T19:15:21,221 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-26T19:15:21,221 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1722): Closing 082e64a1d02e824a02737c5fd026f5bd, disabling compactions & flushes 2024-11-26T19:15:21,221 INFO [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd. 2024-11-26T19:15:21,221 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd. 2024-11-26T19:15:21,221 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd. after waiting 0 ms 2024-11-26T19:15:21,221 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd. 2024-11-26T19:15:21,225 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-26T19:15:21,225 INFO [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd. 2024-11-26T19:15:21,225 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1676): Region close journal for 082e64a1d02e824a02737c5fd026f5bd: Waiting for close lock at 1732648521221Running coprocessor pre-close hooks at 1732648521221Disabling compacts and flushes for region at 1732648521221Disabling writes for close at 1732648521221Writing region close event to WAL at 1732648521222 (+1 ms)Running coprocessor post-close hooks at 1732648521225 (+3 ms)Closed at 1732648521225 2024-11-26T19:15:21,225 WARN [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegionServer(3299): Not adding moved region record: 082e64a1d02e824a02737c5fd026f5bd to self. 2024-11-26T19:15:21,227 INFO [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] handler.UnassignRegionHandler(157): Closed 082e64a1d02e824a02737c5fd026f5bd 2024-11-26T19:15:21,227 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=082e64a1d02e824a02737c5fd026f5bd, regionState=CLOSED 2024-11-26T19:15:21,229 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=89, ppid=88, state=RUNNABLE, hasLock=false; CloseRegionProcedure 082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 because future has completed 2024-11-26T19:15:21,230 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=89, resume processing ppid=88 2024-11-26T19:15:21,230 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, ppid=88, state=SUCCESS, hasLock=false; CloseRegionProcedure 082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 in 160 msec 2024-11-26T19:15:21,231 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=082e64a1d02e824a02737c5fd026f5bd, REOPEN/MOVE; state=CLOSED, location=fe76acf732b2,42785,1732648423970; forceNewPlan=false, retain=true 2024-11-26T19:15:21,381 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=082e64a1d02e824a02737c5fd026f5bd, regionState=OPENING, regionLocation=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:21,383 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=082e64a1d02e824a02737c5fd026f5bd, REOPEN/MOVE because future has completed 2024-11-26T19:15:21,383 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=90, ppid=88, state=RUNNABLE, hasLock=false; OpenRegionProcedure 082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970}] 2024-11-26T19:15:21,538 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd. 2024-11-26T19:15:21,538 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(7752): Opening region: {ENCODED => 082e64a1d02e824a02737c5fd026f5bd, NAME => 'TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd.', STARTKEY => '', ENDKEY => ''} 2024-11-26T19:15:21,538 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 082e64a1d02e824a02737c5fd026f5bd 2024-11-26T19:15:21,538 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T19:15:21,538 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(7794): checking encryption for 082e64a1d02e824a02737c5fd026f5bd 2024-11-26T19:15:21,538 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(7797): checking classloading for 082e64a1d02e824a02737c5fd026f5bd 2024-11-26T19:15:21,539 INFO [StoreOpener-082e64a1d02e824a02737c5fd026f5bd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 082e64a1d02e824a02737c5fd026f5bd 2024-11-26T19:15:21,540 INFO [StoreOpener-082e64a1d02e824a02737c5fd026f5bd-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-26T19:15:21,540 INFO [StoreOpener-082e64a1d02e824a02737c5fd026f5bd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 082e64a1d02e824a02737c5fd026f5bd columnFamilyName A 2024-11-26T19:15:21,541 DEBUG [StoreOpener-082e64a1d02e824a02737c5fd026f5bd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:21,542 INFO [StoreOpener-082e64a1d02e824a02737c5fd026f5bd-1 {}] regionserver.HStore(327): Store=082e64a1d02e824a02737c5fd026f5bd/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T19:15:21,542 INFO [StoreOpener-082e64a1d02e824a02737c5fd026f5bd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 082e64a1d02e824a02737c5fd026f5bd 2024-11-26T19:15:21,543 INFO [StoreOpener-082e64a1d02e824a02737c5fd026f5bd-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-26T19:15:21,543 INFO [StoreOpener-082e64a1d02e824a02737c5fd026f5bd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 082e64a1d02e824a02737c5fd026f5bd columnFamilyName B 2024-11-26T19:15:21,543 DEBUG [StoreOpener-082e64a1d02e824a02737c5fd026f5bd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:21,543 INFO [StoreOpener-082e64a1d02e824a02737c5fd026f5bd-1 {}] regionserver.HStore(327): Store=082e64a1d02e824a02737c5fd026f5bd/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T19:15:21,543 INFO [StoreOpener-082e64a1d02e824a02737c5fd026f5bd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 082e64a1d02e824a02737c5fd026f5bd 2024-11-26T19:15:21,544 INFO [StoreOpener-082e64a1d02e824a02737c5fd026f5bd-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-26T19:15:21,544 INFO [StoreOpener-082e64a1d02e824a02737c5fd026f5bd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 082e64a1d02e824a02737c5fd026f5bd columnFamilyName C 2024-11-26T19:15:21,544 DEBUG [StoreOpener-082e64a1d02e824a02737c5fd026f5bd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:21,544 INFO [StoreOpener-082e64a1d02e824a02737c5fd026f5bd-1 {}] regionserver.HStore(327): Store=082e64a1d02e824a02737c5fd026f5bd/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T19:15:21,544 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd. 2024-11-26T19:15:21,544 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(1038): replaying wal for 082e64a1d02e824a02737c5fd026f5bd 2024-11-26T19:15:21,545 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd 2024-11-26T19:15:21,546 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd 2024-11-26T19:15:21,546 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(1048): stopping wal replay for 082e64a1d02e824a02737c5fd026f5bd 2024-11-26T19:15:21,546 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(1060): Cleaning up temporary data for 082e64a1d02e824a02737c5fd026f5bd 2024-11-26T19:15:21,547 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-26T19:15:21,548 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(1093): writing seq id for 082e64a1d02e824a02737c5fd026f5bd 2024-11-26T19:15:21,548 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(1114): Opened 082e64a1d02e824a02737c5fd026f5bd; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62390857, jitterRate=-0.0703037828207016}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-26T19:15:21,548 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 082e64a1d02e824a02737c5fd026f5bd 2024-11-26T19:15:21,549 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(1006): Region open journal for 082e64a1d02e824a02737c5fd026f5bd: Running coprocessor pre-open hook at 1732648521538Writing region info on filesystem at 1732648521538Initializing all the Stores at 1732648521539 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732648521539Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732648521539Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732648521539Cleaning up temporary data from old regions at 1732648521546 (+7 ms)Running coprocessor post-open hooks at 1732648521548 (+2 ms)Region opened successfully at 1732648521549 (+1 ms) 2024-11-26T19:15:21,550 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., pid=90, masterSystemTime=1732648521535 2024-11-26T19:15:21,551 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd. 2024-11-26T19:15:21,551 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd. 2024-11-26T19:15:21,551 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=082e64a1d02e824a02737c5fd026f5bd, regionState=OPEN, openSeqNum=5, regionLocation=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:21,553 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=90, ppid=88, state=RUNNABLE, hasLock=false; OpenRegionProcedure 082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 because future has completed 2024-11-26T19:15:21,555 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=90, resume processing ppid=88 2024-11-26T19:15:21,555 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, ppid=88, state=SUCCESS, hasLock=false; OpenRegionProcedure 082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 in 170 msec 2024-11-26T19:15:21,556 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=88, resume processing ppid=87 2024-11-26T19:15:21,556 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, ppid=87, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=082e64a1d02e824a02737c5fd026f5bd, REOPEN/MOVE in 489 msec 2024-11-26T19:15:21,558 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=86 2024-11-26T19:15:21,558 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=86, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 492 msec 2024-11-26T19:15:21,559 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 906 msec 2024-11-26T19:15:21,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-26T19:15:21,560 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-26T19:15:21,561 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c06627f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:15:21,561 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fe76acf732b2,33811,-1 for getting cluster id 2024-11-26T19:15:21,561 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T19:15:21,562 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3cec4e24-71db-4cd0-b63a-4ea30a9b160b' 2024-11-26T19:15:21,562 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T19:15:21,562 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3cec4e24-71db-4cd0-b63a-4ea30a9b160b" 2024-11-26T19:15:21,562 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c43e443, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:15:21,562 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fe76acf732b2,33811,-1] 2024-11-26T19:15:21,562 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T19:15:21,563 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:15:21,563 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35088, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T19:15:21,564 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ef95aca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:15:21,564 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e8673cb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:15:21,564 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fe76acf732b2,33811,-1 for getting cluster id 2024-11-26T19:15:21,564 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T19:15:21,565 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3cec4e24-71db-4cd0-b63a-4ea30a9b160b' 2024-11-26T19:15:21,565 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T19:15:21,565 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3cec4e24-71db-4cd0-b63a-4ea30a9b160b" 2024-11-26T19:15:21,565 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6da17c42, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:15:21,565 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fe76acf732b2,33811,-1] 2024-11-26T19:15:21,566 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T19:15:21,566 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:15:21,566 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35114, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T19:15:21,567 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@701764a5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:15:21,567 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18ca1e42, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:15:21,567 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fe76acf732b2,33811,-1 for getting cluster id 2024-11-26T19:15:21,567 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T19:15:21,568 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3cec4e24-71db-4cd0-b63a-4ea30a9b160b' 2024-11-26T19:15:21,568 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T19:15:21,568 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3cec4e24-71db-4cd0-b63a-4ea30a9b160b" 2024-11-26T19:15:21,568 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14131ca3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:15:21,568 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fe76acf732b2,33811,-1] 2024-11-26T19:15:21,569 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T19:15:21,569 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:15:21,569 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35140, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T19:15:21,570 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43ac56e3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:15:21,570 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c7eae17, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:15:21,570 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fe76acf732b2,33811,-1 for getting cluster id 2024-11-26T19:15:21,570 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T19:15:21,571 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3cec4e24-71db-4cd0-b63a-4ea30a9b160b' 2024-11-26T19:15:21,571 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T19:15:21,571 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3cec4e24-71db-4cd0-b63a-4ea30a9b160b" 2024-11-26T19:15:21,571 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9957730, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:15:21,571 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fe76acf732b2,33811,-1] 2024-11-26T19:15:21,571 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T19:15:21,571 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:15:21,572 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35154, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T19:15:21,572 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b3ce665, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:15:21,573 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73c5be4e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:15:21,573 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fe76acf732b2,33811,-1 for getting cluster id 2024-11-26T19:15:21,573 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T19:15:21,573 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3cec4e24-71db-4cd0-b63a-4ea30a9b160b' 2024-11-26T19:15:21,574 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T19:15:21,574 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3cec4e24-71db-4cd0-b63a-4ea30a9b160b" 2024-11-26T19:15:21,574 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79a4041, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:15:21,574 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fe76acf732b2,33811,-1] 2024-11-26T19:15:21,574 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T19:15:21,574 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:15:21,575 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35164, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T19:15:21,575 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5dd4aaeb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:15:21,575 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@395008aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:15:21,576 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fe76acf732b2,33811,-1 for getting cluster id 2024-11-26T19:15:21,576 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T19:15:21,576 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3cec4e24-71db-4cd0-b63a-4ea30a9b160b' 2024-11-26T19:15:21,576 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T19:15:21,576 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3cec4e24-71db-4cd0-b63a-4ea30a9b160b" 2024-11-26T19:15:21,576 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d5e326a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:15:21,576 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fe76acf732b2,33811,-1] 2024-11-26T19:15:21,577 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T19:15:21,577 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:15:21,577 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35188, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T19:15:21,578 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3980d724, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:15:21,578 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61351bc2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:15:21,578 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fe76acf732b2,33811,-1 for getting cluster id 2024-11-26T19:15:21,578 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T19:15:21,579 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3cec4e24-71db-4cd0-b63a-4ea30a9b160b' 2024-11-26T19:15:21,579 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T19:15:21,579 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3cec4e24-71db-4cd0-b63a-4ea30a9b160b" 2024-11-26T19:15:21,579 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e06f847, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:15:21,579 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fe76acf732b2,33811,-1] 2024-11-26T19:15:21,579 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T19:15:21,579 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:15:21,580 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35198, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T19:15:21,580 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1dc5710b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:15:21,581 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6444e4e1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:15:21,581 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fe76acf732b2,33811,-1 for getting cluster id 2024-11-26T19:15:21,581 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T19:15:21,582 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3cec4e24-71db-4cd0-b63a-4ea30a9b160b' 2024-11-26T19:15:21,582 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T19:15:21,582 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3cec4e24-71db-4cd0-b63a-4ea30a9b160b" 2024-11-26T19:15:21,582 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@284fc0f3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:15:21,582 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fe76acf732b2,33811,-1] 2024-11-26T19:15:21,582 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T19:15:21,582 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:15:21,583 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35220, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T19:15:21,583 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b710aa8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:15:21,584 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b350ef2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:15:21,584 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fe76acf732b2,33811,-1 for getting cluster id 2024-11-26T19:15:21,584 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T19:15:21,584 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3cec4e24-71db-4cd0-b63a-4ea30a9b160b' 2024-11-26T19:15:21,584 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T19:15:21,585 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3cec4e24-71db-4cd0-b63a-4ea30a9b160b" 2024-11-26T19:15:21,585 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1da51c47, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:15:21,585 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fe76acf732b2,33811,-1] 2024-11-26T19:15:21,585 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T19:15:21,585 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:15:21,586 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35250, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T19:15:21,586 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b6a39b9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:15:21,586 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@775a762e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:15:21,586 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fe76acf732b2,33811,-1 for getting cluster id 2024-11-26T19:15:21,587 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T19:15:21,587 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3cec4e24-71db-4cd0-b63a-4ea30a9b160b' 2024-11-26T19:15:21,587 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T19:15:21,587 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3cec4e24-71db-4cd0-b63a-4ea30a9b160b" 2024-11-26T19:15:21,588 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79146157, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:15:21,588 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fe76acf732b2,33811,-1] 2024-11-26T19:15:21,588 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T19:15:21,588 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:15:21,588 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35278, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T19:15:21,589 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59d8067e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:15:21,589 DEBUG [Thread-1615 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T19:15:21,589 DEBUG [Thread-1617 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T19:15:21,589 DEBUG [Thread-1609 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T19:15:21,590 DEBUG [Thread-1622 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T19:15:21,590 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fe76acf732b2,42785,1732648423970, seqNum=-1] 2024-11-26T19:15:21,590 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fe76acf732b2,42785,1732648423970, seqNum=-1] 2024-11-26T19:15:21,590 DEBUG [Thread-1611 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T19:15:21,590 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fe76acf732b2,42785,1732648423970, seqNum=-1] 2024-11-26T19:15:21,590 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fe76acf732b2,42785,1732648423970, seqNum=-1] 2024-11-26T19:15:21,590 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T19:15:21,590 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T19:15:21,590 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T19:15:21,591 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T19:15:21,591 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fe76acf732b2,42785,1732648423970, seqNum=-1] 2024-11-26T19:15:21,591 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T19:15:21,591 DEBUG [Thread-1626 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T19:15:21,592 DEBUG [Thread-1620 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T19:15:21,592 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58846, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T19:15:21,592 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fe76acf732b2,42785,1732648423970, seqNum=-1] 2024-11-26T19:15:21,592 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58842, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T19:15:21,592 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58878, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T19:15:21,592 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T19:15:21,592 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58864, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T19:15:21,592 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58858, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T19:15:21,593 DEBUG [Thread-1613 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T19:15:21,593 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fe76acf732b2,42785,1732648423970, seqNum=-1] 2024-11-26T19:15:21,593 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58888, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T19:15:21,593 DEBUG [Thread-1624 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T19:15:21,593 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T19:15:21,593 DEBUG [Thread-1628 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T19:15:21,594 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fe76acf732b2,42785,1732648423970, seqNum=-1] 2024-11-26T19:15:21,594 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58902, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T19:15:21,594 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T19:15:21,594 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fe76acf732b2,42785,1732648423970, seqNum=-1] 2024-11-26T19:15:21,595 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T19:15:21,595 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5] 2024-11-26T19:15:21,595 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5] 2024-11-26T19:15:21,596 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fe76acf732b2,42785,1732648423970, seqNum=-1] 2024-11-26T19:15:21,596 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T19:15:21,596 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5] 2024-11-26T19:15:21,597 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58914, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T19:15:21,597 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5] 2024-11-26T19:15:21,597 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58942, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T19:15:21,597 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58928, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T19:15:21,598 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5] 2024-11-26T19:15:21,598 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5] 2024-11-26T19:15:21,599 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5] 2024-11-26T19:15:21,600 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33811 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T19:15:21,600 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5] 2024-11-26T19:15:21,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33811 {}] procedure2.ProcedureExecutor(1139): Stored pid=91, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees 2024-11-26T19:15:21,603 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5] 2024-11-26T19:15:21,603 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5] 2024-11-26T19:15:21,603 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=91, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T19:15:21,604 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=91, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T19:15:21,604 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=92, ppid=91, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T19:15:21,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(8855): Flush requested on 082e64a1d02e824a02737c5fd026f5bd 2024-11-26T19:15:21,606 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 082e64a1d02e824a02737c5fd026f5bd 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-26T19:15:21,607 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 082e64a1d02e824a02737c5fd026f5bd, store=A 2024-11-26T19:15:21,607 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:15:21,607 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 082e64a1d02e824a02737c5fd026f5bd, store=B 2024-11-26T19:15:21,607 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:15:21,607 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 082e64a1d02e824a02737c5fd026f5bd, store=C 2024-11-26T19:15:21,607 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:15:21,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-26T19:15:21,636 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112605d6bdef05c24a409d95833276c93804_082e64a1d02e824a02737c5fd026f5bd is 50, key is test_row_0/A:col10/1732648521605/Put/seqid=0 2024-11-26T19:15:21,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742180_1356 (size=14594) 2024-11-26T19:15:21,642 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:21,646 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112605d6bdef05c24a409d95833276c93804_082e64a1d02e824a02737c5fd026f5bd to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112605d6bdef05c24a409d95833276c93804_082e64a1d02e824a02737c5fd026f5bd 2024-11-26T19:15:21,646 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp/A/aa10863e62b74d7a963b18b94263ba99, store: [table=TestAcidGuarantees family=A region=082e64a1d02e824a02737c5fd026f5bd] 2024-11-26T19:15:21,647 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp/A/aa10863e62b74d7a963b18b94263ba99 is 175, key is test_row_0/A:col10/1732648521605/Put/seqid=0 2024-11-26T19:15:21,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742181_1357 (size=39549) 2024-11-26T19:15:21,652 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:21,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58842 deadline: 1732648581639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:21,652 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:21,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58864 deadline: 1732648581642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:21,652 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:21,652 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:21,652 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:21,653 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:21,653 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:21,653 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:21,654 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732648521635, max timestamp=1732648521650Num uniques 859; 2024-11-26T19:15:21,654 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-26T19:15:21,654 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732648521635, max timestamp=1732648521650Num uniques 859; 2024-11-26T19:15:21,654 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-26T19:15:21,655 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732648521635, max timestamp=1732648521650Num uniques 859; 2024-11-26T19:15:21,655 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-26T19:15:21,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-26T19:15:21,712 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:21,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58878 deadline: 1732648581709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:21,713 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:21,713 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:21,713 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:21,724 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:21,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58914 deadline: 1732648581711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:21,725 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:21,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58858 deadline: 1732648581711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:21,725 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:21,725 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:21,725 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:21,725 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:21,725 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:21,725 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:21,757 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-26T19:15:21,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd. 2024-11-26T19:15:21,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd. as already flushing 2024-11-26T19:15:21,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd. 2024-11-26T19:15:21,758 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 082e64a1d02e824a02737c5fd026f5bd, NAME => 'TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:21,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 082e64a1d02e824a02737c5fd026f5bd, NAME => 'TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:21,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 082e64a1d02e824a02737c5fd026f5bd, NAME => 'TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 082e64a1d02e824a02737c5fd026f5bd, NAME => 'TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:21,770 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:21,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58842 deadline: 1732648581761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:21,770 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:21,770 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:21,770 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:21,770 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:21,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58864 deadline: 1732648581761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:21,771 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:21,771 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:21,771 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:21,829 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:21,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58878 deadline: 1732648581822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:21,830 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:21,830 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:21,830 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:21,844 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:21,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58914 deadline: 1732648581831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:21,845 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:21,845 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:21,845 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:21,845 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:21,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58858 deadline: 1732648581831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:21,845 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:21,845 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:21,845 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:21,910 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-26T19:15:21,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd. 2024-11-26T19:15:21,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd. as already flushing 2024-11-26T19:15:21,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd. 2024-11-26T19:15:21,910 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 082e64a1d02e824a02737c5fd026f5bd, NAME => 'TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:21,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 082e64a1d02e824a02737c5fd026f5bd, NAME => 'TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:21,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 082e64a1d02e824a02737c5fd026f5bd, NAME => 'TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 082e64a1d02e824a02737c5fd026f5bd, NAME => 'TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:21,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-26T19:15:21,989 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:21,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58842 deadline: 1732648581982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:21,990 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:21,990 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:21,990 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:21,990 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:21,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58864 deadline: 1732648581982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:21,990 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:21,990 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:21,990 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:22,048 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:22,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58878 deadline: 1732648582041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:22,048 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:22,048 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:22,049 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:22,052 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp/A/aa10863e62b74d7a963b18b94263ba99 2024-11-26T19:15:22,062 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:22,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58914 deadline: 1732648582052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:22,063 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:22,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58858 deadline: 1732648582052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:22,063 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-26T19:15:22,063 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:22,063 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:22,063 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:22,063 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:22,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd. 2024-11-26T19:15:22,063 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:22,063 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:22,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd. as already flushing 2024-11-26T19:15:22,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd. 2024-11-26T19:15:22,063 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 082e64a1d02e824a02737c5fd026f5bd, NAME => 'TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:22,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 082e64a1d02e824a02737c5fd026f5bd, NAME => 'TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:22,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 082e64a1d02e824a02737c5fd026f5bd, NAME => 'TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 082e64a1d02e824a02737c5fd026f5bd, NAME => 'TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:22,077 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp/B/6077bbc4704c4f8aaf3450bda07d63ea is 50, key is test_row_0/B:col10/1732648521605/Put/seqid=0 2024-11-26T19:15:22,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742182_1358 (size=12001) 2024-11-26T19:15:22,217 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-26T19:15:22,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd. 2024-11-26T19:15:22,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd. as already flushing 2024-11-26T19:15:22,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd. 2024-11-26T19:15:22,217 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 082e64a1d02e824a02737c5fd026f5bd, NAME => 'TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:22,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 082e64a1d02e824a02737c5fd026f5bd, NAME => 'TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:22,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 082e64a1d02e824a02737c5fd026f5bd, NAME => 'TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 082e64a1d02e824a02737c5fd026f5bd, NAME => 'TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:22,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-26T19:15:22,312 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:22,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58842 deadline: 1732648582302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:22,312 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:22,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58864 deadline: 1732648582302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:22,313 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:22,313 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:22,313 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:22,313 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:22,313 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:22,313 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:22,364 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:22,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58878 deadline: 1732648582352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:22,364 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:22,364 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:22,364 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:22,370 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-26T19:15:22,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd. 2024-11-26T19:15:22,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd. as already flushing 2024-11-26T19:15:22,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd. 2024-11-26T19:15:22,370 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 082e64a1d02e824a02737c5fd026f5bd, NAME => 'TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:22,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 082e64a1d02e824a02737c5fd026f5bd, NAME => 'TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:22,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 082e64a1d02e824a02737c5fd026f5bd, NAME => 'TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 082e64a1d02e824a02737c5fd026f5bd, NAME => 'TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:22,379 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:22,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58914 deadline: 1732648582372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:22,379 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:22,379 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:22,380 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:22,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58858 deadline: 1732648582372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:22,380 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:22,380 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:22,380 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:22,380 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:22,483 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp/B/6077bbc4704c4f8aaf3450bda07d63ea 2024-11-26T19:15:22,507 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp/C/5a09b4d5444b4a6c8039aadea772b401 is 50, key is test_row_0/C:col10/1732648521605/Put/seqid=0 2024-11-26T19:15:22,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742183_1359 (size=12001) 2024-11-26T19:15:22,523 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-26T19:15:22,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd. 2024-11-26T19:15:22,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd. as already flushing 2024-11-26T19:15:22,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd. 2024-11-26T19:15:22,523 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 082e64a1d02e824a02737c5fd026f5bd, NAME => 'TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:22,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 082e64a1d02e824a02737c5fd026f5bd, NAME => 'TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:22,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 082e64a1d02e824a02737c5fd026f5bd, NAME => 'TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 082e64a1d02e824a02737c5fd026f5bd, NAME => 'TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:22,676 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-26T19:15:22,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd. 2024-11-26T19:15:22,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd. as already flushing 2024-11-26T19:15:22,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd. 2024-11-26T19:15:22,677 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 082e64a1d02e824a02737c5fd026f5bd, NAME => 'TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:22,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 082e64a1d02e824a02737c5fd026f5bd, NAME => 'TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:22,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 082e64a1d02e824a02737c5fd026f5bd, NAME => 'TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 082e64a1d02e824a02737c5fd026f5bd, NAME => 'TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:22,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-26T19:15:22,829 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-26T19:15:22,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd. 2024-11-26T19:15:22,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd. as already flushing 2024-11-26T19:15:22,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd. 2024-11-26T19:15:22,830 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 082e64a1d02e824a02737c5fd026f5bd, NAME => 'TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:22,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 082e64a1d02e824a02737c5fd026f5bd, NAME => 'TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:22,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 082e64a1d02e824a02737c5fd026f5bd, NAME => 'TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 082e64a1d02e824a02737c5fd026f5bd, NAME => 'TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:22,831 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:22,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58842 deadline: 1732648582821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:22,831 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:22,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58864 deadline: 1732648582822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:22,832 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:22,832 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:22,832 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:22,832 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:22,832 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:22,832 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:22,884 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:22,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58878 deadline: 1732648582872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:22,884 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:22,884 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:22,884 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:22,898 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:22,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58914 deadline: 1732648582891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:22,899 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:22,899 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:22,899 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:22,899 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:22,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58858 deadline: 1732648582891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:22,899 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:22,899 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:22,899 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:22,912 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp/C/5a09b4d5444b4a6c8039aadea772b401 2024-11-26T19:15:22,916 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp/A/aa10863e62b74d7a963b18b94263ba99 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/A/aa10863e62b74d7a963b18b94263ba99 2024-11-26T19:15:22,920 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/A/aa10863e62b74d7a963b18b94263ba99, entries=200, sequenceid=15, filesize=38.6 K 2024-11-26T19:15:22,921 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp/B/6077bbc4704c4f8aaf3450bda07d63ea as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/B/6077bbc4704c4f8aaf3450bda07d63ea 2024-11-26T19:15:22,925 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/B/6077bbc4704c4f8aaf3450bda07d63ea, entries=150, sequenceid=15, filesize=11.7 K 2024-11-26T19:15:22,926 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp/C/5a09b4d5444b4a6c8039aadea772b401 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/C/5a09b4d5444b4a6c8039aadea772b401 2024-11-26T19:15:22,929 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/C/5a09b4d5444b4a6c8039aadea772b401, entries=150, sequenceid=15, filesize=11.7 K 2024-11-26T19:15:22,930 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for 082e64a1d02e824a02737c5fd026f5bd in 1325ms, sequenceid=15, compaction requested=false 2024-11-26T19:15:22,930 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 082e64a1d02e824a02737c5fd026f5bd: 2024-11-26T19:15:22,983 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-26T19:15:22,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd. 2024-11-26T19:15:22,983 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2902): Flushing 082e64a1d02e824a02737c5fd026f5bd 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-11-26T19:15:22,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 082e64a1d02e824a02737c5fd026f5bd, store=A 2024-11-26T19:15:22,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T19:15:22,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 082e64a1d02e824a02737c5fd026f5bd, store=B 2024-11-26T19:15:22,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T19:15:22,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 082e64a1d02e824a02737c5fd026f5bd, store=C 2024-11-26T19:15:22,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T19:15:22,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126f33412f9316b4af487df3679eb452a37_082e64a1d02e824a02737c5fd026f5bd is 50, key is test_row_0/A:col10/1732648521696/Put/seqid=0 2024-11-26T19:15:22,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742184_1360 (size=12154) 2024-11-26T19:15:22,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:23,000 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126f33412f9316b4af487df3679eb452a37_082e64a1d02e824a02737c5fd026f5bd to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126f33412f9316b4af487df3679eb452a37_082e64a1d02e824a02737c5fd026f5bd 2024-11-26T19:15:23,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp/A/f6cd4e46b01d4402b7ea76d3df1439e6, store: [table=TestAcidGuarantees family=A region=082e64a1d02e824a02737c5fd026f5bd] 2024-11-26T19:15:23,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp/A/f6cd4e46b01d4402b7ea76d3df1439e6 is 175, key is test_row_0/A:col10/1732648521696/Put/seqid=0 2024-11-26T19:15:23,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742185_1361 (size=30955) 2024-11-26T19:15:23,413 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=50, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp/A/f6cd4e46b01d4402b7ea76d3df1439e6 2024-11-26T19:15:23,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp/B/856a61e532ef4b4c99f52e52902bbf15 is 50, key is test_row_0/B:col10/1732648521696/Put/seqid=0 2024-11-26T19:15:23,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742186_1362 (size=12001) 2024-11-26T19:15:23,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-26T19:15:23,826 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp/B/856a61e532ef4b4c99f52e52902bbf15 2024-11-26T19:15:23,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp/C/a547238000ac4e9b8d8eebedee916dbd is 50, key is test_row_0/C:col10/1732648521696/Put/seqid=0 2024-11-26T19:15:23,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742187_1363 (size=12001) 2024-11-26T19:15:23,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(8855): Flush requested on 082e64a1d02e824a02737c5fd026f5bd 2024-11-26T19:15:23,847 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd. as already flushing 2024-11-26T19:15:23,891 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:23,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58864 deadline: 1732648583876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:23,891 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:23,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58842 deadline: 1732648583877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:23,892 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:23,892 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:23,892 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:23,892 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:23,892 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:23,892 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:23,897 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:23,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58878 deadline: 1732648583892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:23,898 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to fe76acf732b2,42785,1732648423970 for 'test_row_0' in 082e64a1d02e824a02737c5fd026f5bd of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2188 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T19:15:23,898 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:23,898 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:23,898 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:23,922 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:23,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58914 deadline: 1732648583911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:23,922 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:23,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58858 deadline: 1732648583911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:23,922 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to fe76acf732b2,42785,1732648423970 for 'test_row_0' in 082e64a1d02e824a02737c5fd026f5bd of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2211 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T19:15:23,923 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:23,922 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to fe76acf732b2,42785,1732648423970 for 'test_row_2' in 082e64a1d02e824a02737c5fd026f5bd of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2211 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T19:15:23,923 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:23,923 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:23,923 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:23,923 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:23,923 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:24,011 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:24,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58864 deadline: 1732648584001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:24,012 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:24,012 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:24,012 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:24,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58842 deadline: 1732648584001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:24,012 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:24,012 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:24,012 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:24,012 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:24,231 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:24,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58864 deadline: 1732648584222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:24,232 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:24,231 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:24,232 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:24,232 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:24,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58842 deadline: 1732648584222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:24,232 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:24,232 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:24,232 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:24,238 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp/C/a547238000ac4e9b8d8eebedee916dbd 2024-11-26T19:15:24,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp/A/f6cd4e46b01d4402b7ea76d3df1439e6 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/A/f6cd4e46b01d4402b7ea76d3df1439e6 2024-11-26T19:15:24,247 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/A/f6cd4e46b01d4402b7ea76d3df1439e6, entries=150, sequenceid=50, filesize=30.2 K 2024-11-26T19:15:24,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp/B/856a61e532ef4b4c99f52e52902bbf15 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/B/856a61e532ef4b4c99f52e52902bbf15 2024-11-26T19:15:24,253 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/B/856a61e532ef4b4c99f52e52902bbf15, entries=150, sequenceid=50, filesize=11.7 K 2024-11-26T19:15:24,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp/C/a547238000ac4e9b8d8eebedee916dbd as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/C/a547238000ac4e9b8d8eebedee916dbd 2024-11-26T19:15:24,257 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/C/a547238000ac4e9b8d8eebedee916dbd, entries=150, sequenceid=50, filesize=11.7 K 2024-11-26T19:15:24,257 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=46.96 KB/48090 for 082e64a1d02e824a02737c5fd026f5bd in 1274ms, sequenceid=50, compaction requested=false 2024-11-26T19:15:24,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2603): Flush status journal for 082e64a1d02e824a02737c5fd026f5bd: 2024-11-26T19:15:24,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd. 2024-11-26T19:15:24,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=92 2024-11-26T19:15:24,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33811 {}] master.HMaster(4169): Remote procedure done, pid=92 2024-11-26T19:15:24,261 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=92, resume processing ppid=91 2024-11-26T19:15:24,261 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, ppid=91, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6550 sec 2024-11-26T19:15:24,262 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees in 2.6610 sec 2024-11-26T19:15:24,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(8855): Flush requested on 082e64a1d02e824a02737c5fd026f5bd 2024-11-26T19:15:24,546 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 082e64a1d02e824a02737c5fd026f5bd 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-26T19:15:24,547 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 082e64a1d02e824a02737c5fd026f5bd, store=A 2024-11-26T19:15:24,547 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:15:24,547 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 082e64a1d02e824a02737c5fd026f5bd, store=B 2024-11-26T19:15:24,547 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:15:24,547 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 082e64a1d02e824a02737c5fd026f5bd, store=C 2024-11-26T19:15:24,547 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:15:24,555 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112655cdd69bebbc4313ae48cf1b1a798a68_082e64a1d02e824a02737c5fd026f5bd is 50, key is test_row_0/A:col10/1732648524545/Put/seqid=0 2024-11-26T19:15:24,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742188_1364 (size=14594) 2024-11-26T19:15:24,565 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:24,569 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112655cdd69bebbc4313ae48cf1b1a798a68_082e64a1d02e824a02737c5fd026f5bd to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112655cdd69bebbc4313ae48cf1b1a798a68_082e64a1d02e824a02737c5fd026f5bd 2024-11-26T19:15:24,573 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp/A/2bbe6875e5f4473ca1eaeeb78a7e7609, store: [table=TestAcidGuarantees family=A region=082e64a1d02e824a02737c5fd026f5bd] 2024-11-26T19:15:24,573 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp/A/2bbe6875e5f4473ca1eaeeb78a7e7609 is 175, key is test_row_0/A:col10/1732648524545/Put/seqid=0 2024-11-26T19:15:24,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742189_1365 (size=39549) 2024-11-26T19:15:24,586 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=61, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp/A/2bbe6875e5f4473ca1eaeeb78a7e7609 2024-11-26T19:15:24,597 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp/B/1c6f109693564675b6ade8075652de00 is 50, key is test_row_0/B:col10/1732648524545/Put/seqid=0 2024-11-26T19:15:24,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742190_1366 (size=12001) 2024-11-26T19:15:24,658 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732648524614, max timestamp=1732648524651Num uniques 859; 2024-11-26T19:15:24,658 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-26T19:15:24,658 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732648524614, max timestamp=1732648524651Num uniques 859; 2024-11-26T19:15:24,658 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-26T19:15:24,658 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732648524614, max timestamp=1732648524651Num uniques 859; 2024-11-26T19:15:24,658 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-26T19:15:24,709 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-26T19:15:24,769 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:24,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58864 deadline: 1732648584751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:24,770 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:24,770 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:24,770 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:24,777 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:24,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58842 deadline: 1732648584763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:24,778 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:24,778 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:24,778 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:24,887 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:24,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58864 deadline: 1732648584871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:24,888 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:24,888 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:24,888 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:24,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58842 deadline: 1732648584881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:24,888 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:24,888 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:24,888 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:24,888 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:25,007 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp/B/1c6f109693564675b6ade8075652de00 2024-11-26T19:15:25,015 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp/C/4eeca70933ea407ea9ab96d857e3db89 is 50, key is test_row_0/C:col10/1732648524545/Put/seqid=0 2024-11-26T19:15:25,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742191_1367 (size=12001) 2024-11-26T19:15:25,106 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:25,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58864 deadline: 1732648585092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:25,106 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:25,107 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:25,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58842 deadline: 1732648585092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:25,107 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:25,107 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:25,107 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:25,107 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:25,107 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:25,420 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp/C/4eeca70933ea407ea9ab96d857e3db89 2024-11-26T19:15:25,423 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:25,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58864 deadline: 1732648585412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:25,423 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:25,423 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:25,423 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:25,423 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:25,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58842 deadline: 1732648585412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:25,423 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:25,423 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:25,423 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:25,424 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp/A/2bbe6875e5f4473ca1eaeeb78a7e7609 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/A/2bbe6875e5f4473ca1eaeeb78a7e7609 2024-11-26T19:15:25,428 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/A/2bbe6875e5f4473ca1eaeeb78a7e7609, entries=200, sequenceid=61, filesize=38.6 K 2024-11-26T19:15:25,429 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp/B/1c6f109693564675b6ade8075652de00 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/B/1c6f109693564675b6ade8075652de00 2024-11-26T19:15:25,433 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/B/1c6f109693564675b6ade8075652de00, entries=150, sequenceid=61, filesize=11.7 K 2024-11-26T19:15:25,434 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp/C/4eeca70933ea407ea9ab96d857e3db89 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/C/4eeca70933ea407ea9ab96d857e3db89 2024-11-26T19:15:25,437 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/C/4eeca70933ea407ea9ab96d857e3db89, entries=150, sequenceid=61, filesize=11.7 K 2024-11-26T19:15:25,438 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 082e64a1d02e824a02737c5fd026f5bd in 892ms, sequenceid=61, compaction requested=true 2024-11-26T19:15:25,438 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 082e64a1d02e824a02737c5fd026f5bd: 2024-11-26T19:15:25,438 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 082e64a1d02e824a02737c5fd026f5bd:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T19:15:25,438 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:15:25,438 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T19:15:25,438 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T19:15:25,438 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 082e64a1d02e824a02737c5fd026f5bd:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T19:15:25,439 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:15:25,439 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 082e64a1d02e824a02737c5fd026f5bd:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T19:15:25,439 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T19:15:25,439 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T19:15:25,439 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110053 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T19:15:25,439 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1541): 082e64a1d02e824a02737c5fd026f5bd/B is initiating minor compaction (all files) 2024-11-26T19:15:25,439 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1541): 082e64a1d02e824a02737c5fd026f5bd/A is initiating minor compaction (all files) 2024-11-26T19:15:25,439 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 082e64a1d02e824a02737c5fd026f5bd/B in TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd. 2024-11-26T19:15:25,440 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 082e64a1d02e824a02737c5fd026f5bd/A in TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd. 2024-11-26T19:15:25,440 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/B/6077bbc4704c4f8aaf3450bda07d63ea, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/B/856a61e532ef4b4c99f52e52902bbf15, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/B/1c6f109693564675b6ade8075652de00] into tmpdir=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp, totalSize=35.2 K 2024-11-26T19:15:25,440 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/A/aa10863e62b74d7a963b18b94263ba99, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/A/f6cd4e46b01d4402b7ea76d3df1439e6, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/A/2bbe6875e5f4473ca1eaeeb78a7e7609] into tmpdir=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp, totalSize=107.5 K 2024-11-26T19:15:25,440 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd. 2024-11-26T19:15:25,440 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd. files: [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/A/aa10863e62b74d7a963b18b94263ba99, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/A/f6cd4e46b01d4402b7ea76d3df1439e6, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/A/2bbe6875e5f4473ca1eaeeb78a7e7609] 2024-11-26T19:15:25,441 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting 6077bbc4704c4f8aaf3450bda07d63ea, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732648521602 2024-11-26T19:15:25,441 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting aa10863e62b74d7a963b18b94263ba99, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732648521599 2024-11-26T19:15:25,441 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting f6cd4e46b01d4402b7ea76d3df1439e6, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732648521696 2024-11-26T19:15:25,441 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting 856a61e532ef4b4c99f52e52902bbf15, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732648521696 2024-11-26T19:15:25,442 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2bbe6875e5f4473ca1eaeeb78a7e7609, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732648523860 2024-11-26T19:15:25,442 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting 1c6f109693564675b6ade8075652de00, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732648523860 2024-11-26T19:15:25,448 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=082e64a1d02e824a02737c5fd026f5bd] 2024-11-26T19:15:25,450 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 082e64a1d02e824a02737c5fd026f5bd#B#compaction#312 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T19:15:25,451 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp/B/9e574ca652374eceaed7ac1547ce6564 is 50, key is test_row_0/B:col10/1732648524545/Put/seqid=0 2024-11-26T19:15:25,454 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241126e8c91d62da1443fda5731df6b403a397_082e64a1d02e824a02737c5fd026f5bd store=[table=TestAcidGuarantees family=A region=082e64a1d02e824a02737c5fd026f5bd] 2024-11-26T19:15:25,456 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241126e8c91d62da1443fda5731df6b403a397_082e64a1d02e824a02737c5fd026f5bd, store=[table=TestAcidGuarantees family=A region=082e64a1d02e824a02737c5fd026f5bd] 2024-11-26T19:15:25,456 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126e8c91d62da1443fda5731df6b403a397_082e64a1d02e824a02737c5fd026f5bd because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=082e64a1d02e824a02737c5fd026f5bd] 2024-11-26T19:15:25,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742192_1368 (size=12104) 2024-11-26T19:15:25,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742193_1369 (size=4469) 2024-11-26T19:15:25,463 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 082e64a1d02e824a02737c5fd026f5bd#A#compaction#311 average throughput is 1.63 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T19:15:25,463 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp/A/f3ca20aed6e5408abee89210f3ffa1d9 is 175, key is test_row_0/A:col10/1732648524545/Put/seqid=0 2024-11-26T19:15:25,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742194_1370 (size=31058) 2024-11-26T19:15:25,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-26T19:15:25,762 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T19:15:25,762 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33811 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T19:15:25,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33811 {}] procedure2.ProcedureExecutor(1139): Stored pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees 2024-11-26T19:15:25,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-26T19:15:25,764 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T19:15:25,765 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T19:15:25,765 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T19:15:25,863 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp/B/9e574ca652374eceaed7ac1547ce6564 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/B/9e574ca652374eceaed7ac1547ce6564 2024-11-26T19:15:25,867 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 082e64a1d02e824a02737c5fd026f5bd/B of 082e64a1d02e824a02737c5fd026f5bd into 9e574ca652374eceaed7ac1547ce6564(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T19:15:25,867 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 082e64a1d02e824a02737c5fd026f5bd: 2024-11-26T19:15:25,867 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., storeName=082e64a1d02e824a02737c5fd026f5bd/B, priority=13, startTime=1732648525438; duration=0sec 2024-11-26T19:15:25,867 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T19:15:25,868 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 082e64a1d02e824a02737c5fd026f5bd:B 2024-11-26T19:15:25,868 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T19:15:25,869 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T19:15:25,869 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1541): 082e64a1d02e824a02737c5fd026f5bd/C is initiating minor compaction (all files) 2024-11-26T19:15:25,869 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 082e64a1d02e824a02737c5fd026f5bd/C in TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd. 2024-11-26T19:15:25,869 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/C/5a09b4d5444b4a6c8039aadea772b401, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/C/a547238000ac4e9b8d8eebedee916dbd, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/C/4eeca70933ea407ea9ab96d857e3db89] into tmpdir=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp, totalSize=35.2 K 2024-11-26T19:15:25,869 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting 5a09b4d5444b4a6c8039aadea772b401, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732648521602 2024-11-26T19:15:25,870 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting a547238000ac4e9b8d8eebedee916dbd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732648521696 2024-11-26T19:15:25,871 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting 4eeca70933ea407ea9ab96d857e3db89, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732648523860 2024-11-26T19:15:25,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-26T19:15:25,873 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp/A/f3ca20aed6e5408abee89210f3ffa1d9 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/A/f3ca20aed6e5408abee89210f3ffa1d9 2024-11-26T19:15:25,879 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 082e64a1d02e824a02737c5fd026f5bd/A of 082e64a1d02e824a02737c5fd026f5bd into f3ca20aed6e5408abee89210f3ffa1d9(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T19:15:25,879 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 082e64a1d02e824a02737c5fd026f5bd: 2024-11-26T19:15:25,879 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., storeName=082e64a1d02e824a02737c5fd026f5bd/A, priority=13, startTime=1732648525438; duration=0sec 2024-11-26T19:15:25,879 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:15:25,879 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 082e64a1d02e824a02737c5fd026f5bd:A 2024-11-26T19:15:25,880 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 082e64a1d02e824a02737c5fd026f5bd#C#compaction#313 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T19:15:25,880 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp/C/2d9fbfcec4ef4a38b1a508e06f99281e is 50, key is test_row_0/C:col10/1732648524545/Put/seqid=0 2024-11-26T19:15:25,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742195_1371 (size=12104) 2024-11-26T19:15:25,890 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp/C/2d9fbfcec4ef4a38b1a508e06f99281e as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/C/2d9fbfcec4ef4a38b1a508e06f99281e 2024-11-26T19:15:25,894 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 082e64a1d02e824a02737c5fd026f5bd/C of 082e64a1d02e824a02737c5fd026f5bd into 2d9fbfcec4ef4a38b1a508e06f99281e(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T19:15:25,894 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 082e64a1d02e824a02737c5fd026f5bd: 2024-11-26T19:15:25,894 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., storeName=082e64a1d02e824a02737c5fd026f5bd/C, priority=13, startTime=1732648525439; duration=0sec 2024-11-26T19:15:25,894 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:15:25,894 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 082e64a1d02e824a02737c5fd026f5bd:C 2024-11-26T19:15:25,917 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-26T19:15:25,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd. 2024-11-26T19:15:25,917 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2902): Flushing 082e64a1d02e824a02737c5fd026f5bd 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-11-26T19:15:25,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 082e64a1d02e824a02737c5fd026f5bd, store=A 2024-11-26T19:15:25,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T19:15:25,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 082e64a1d02e824a02737c5fd026f5bd, store=B 2024-11-26T19:15:25,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T19:15:25,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 082e64a1d02e824a02737c5fd026f5bd, store=C 2024-11-26T19:15:25,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T19:15:25,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(8855): Flush requested on 082e64a1d02e824a02737c5fd026f5bd 2024-11-26T19:15:25,920 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd. as already flushing 2024-11-26T19:15:25,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126add97a30b4d94d8c910c1e1d099758b9_082e64a1d02e824a02737c5fd026f5bd is 50, key is test_row_0/A:col10/1732648524749/Put/seqid=0 2024-11-26T19:15:25,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742196_1372 (size=12154) 2024-11-26T19:15:25,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:25,934 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126add97a30b4d94d8c910c1e1d099758b9_082e64a1d02e824a02737c5fd026f5bd to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126add97a30b4d94d8c910c1e1d099758b9_082e64a1d02e824a02737c5fd026f5bd 2024-11-26T19:15:25,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp/A/cec973f41b5643919ab55d6facce6761, store: [table=TestAcidGuarantees family=A region=082e64a1d02e824a02737c5fd026f5bd] 2024-11-26T19:15:25,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp/A/cec973f41b5643919ab55d6facce6761 is 175, key is test_row_0/A:col10/1732648524749/Put/seqid=0 2024-11-26T19:15:25,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742197_1373 (size=30955) 2024-11-26T19:15:25,957 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:25,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58842 deadline: 1732648585944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:25,958 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:25,958 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:25,958 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:25,973 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:25,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58878 deadline: 1732648585955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:25,973 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:25,973 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:25,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58858 deadline: 1732648585957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:25,974 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:25,974 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:25,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58914 deadline: 1732648585958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:25,974 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:25,974 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:25,974 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:25,974 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:25,974 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:25,974 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:25,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58864 deadline: 1732648585958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:25,974 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:25,974 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:25,975 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:25,975 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:25,975 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:26,075 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:26,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58842 deadline: 1732648586062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:26,075 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:26,075 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:26,075 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:26,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-26T19:15:26,092 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:26,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58878 deadline: 1732648586082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:26,092 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:26,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58858 deadline: 1732648586082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:26,092 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:26,092 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:26,092 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:26,092 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:26,092 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:26,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58914 deadline: 1732648586082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:26,092 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:26,092 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:26,093 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:26,093 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:26,093 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:26,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58864 deadline: 1732648586082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:26,093 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:26,093 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:26,093 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:26,093 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:26,289 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:26,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58842 deadline: 1732648586282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:26,290 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:26,290 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:26,290 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:26,306 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:26,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58878 deadline: 1732648586301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:26,306 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:26,306 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:26,306 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:26,306 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:26,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58858 deadline: 1732648586301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:26,306 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:26,307 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:26,307 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:26,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58914 deadline: 1732648586301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:26,307 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:26,307 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:26,307 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:26,307 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:26,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58864 deadline: 1732648586302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:26,307 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:26,307 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:26,307 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:26,307 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:26,340 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=99, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp/A/cec973f41b5643919ab55d6facce6761 2024-11-26T19:15:26,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp/B/699b2dcd81384c4c839aba9062adebee is 50, key is test_row_0/B:col10/1732648524749/Put/seqid=0 2024-11-26T19:15:26,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742198_1374 (size=12001) 2024-11-26T19:15:26,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-26T19:15:26,615 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:26,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58842 deadline: 1732648586602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:26,615 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:26,615 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:26,615 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:26,624 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:26,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58878 deadline: 1732648586611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:26,625 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:26,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58858 deadline: 1732648586611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:26,625 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:26,625 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:26,625 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:26,625 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:26,625 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:26,625 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:26,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58914 deadline: 1732648586612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:26,625 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:26,626 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:26,626 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:26,626 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:26,626 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:26,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58864 deadline: 1732648586612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:26,626 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:26,626 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:26,626 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:26,754 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=99 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp/B/699b2dcd81384c4c839aba9062adebee 2024-11-26T19:15:26,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp/C/4c85c64e413b422197212ce46d79ad66 is 50, key is test_row_0/C:col10/1732648524749/Put/seqid=0 2024-11-26T19:15:26,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742199_1375 (size=12001) 2024-11-26T19:15:26,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-26T19:15:27,136 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:27,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58842 deadline: 1732648587121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:27,136 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:27,136 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:27,136 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:27,137 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:27,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58878 deadline: 1732648587132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:27,138 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:27,138 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:27,138 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:27,138 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:27,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58858 deadline: 1732648587132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:27,138 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:27,138 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:27,138 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:27,138 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:27,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58914 deadline: 1732648587132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:27,138 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:27,138 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:27,138 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:27,138 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:27,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58864 deadline: 1732648587132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:27,139 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:27,139 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:27,139 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:27,166 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=99 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp/C/4c85c64e413b422197212ce46d79ad66 2024-11-26T19:15:27,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp/A/cec973f41b5643919ab55d6facce6761 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/A/cec973f41b5643919ab55d6facce6761 2024-11-26T19:15:27,175 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/A/cec973f41b5643919ab55d6facce6761, entries=150, sequenceid=99, filesize=30.2 K 2024-11-26T19:15:27,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp/B/699b2dcd81384c4c839aba9062adebee as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/B/699b2dcd81384c4c839aba9062adebee 2024-11-26T19:15:27,179 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/B/699b2dcd81384c4c839aba9062adebee, entries=150, sequenceid=99, filesize=11.7 K 2024-11-26T19:15:27,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp/C/4c85c64e413b422197212ce46d79ad66 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/C/4c85c64e413b422197212ce46d79ad66 2024-11-26T19:15:27,183 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/C/4c85c64e413b422197212ce46d79ad66, entries=150, sequenceid=99, filesize=11.7 K 2024-11-26T19:15:27,184 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=60.38 KB/61830 for 082e64a1d02e824a02737c5fd026f5bd in 1267ms, sequenceid=99, compaction requested=false 2024-11-26T19:15:27,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2603): Flush status journal for 082e64a1d02e824a02737c5fd026f5bd: 2024-11-26T19:15:27,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd. 2024-11-26T19:15:27,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=94 2024-11-26T19:15:27,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33811 {}] master.HMaster(4169): Remote procedure done, pid=94 2024-11-26T19:15:27,187 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=93 2024-11-26T19:15:27,187 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4210 sec 2024-11-26T19:15:27,189 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees in 1.4260 sec 2024-11-26T19:15:27,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-26T19:15:27,912 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T19:15:27,912 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33811 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T19:15:27,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33811 {}] procedure2.ProcedureExecutor(1139): Stored pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees 2024-11-26T19:15:27,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-26T19:15:27,914 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T19:15:27,915 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T19:15:27,915 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T19:15:28,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-26T19:15:28,067 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-26T19:15:28,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd. 2024-11-26T19:15:28,068 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2902): Flushing 082e64a1d02e824a02737c5fd026f5bd 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-26T19:15:28,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 082e64a1d02e824a02737c5fd026f5bd, store=A 2024-11-26T19:15:28,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:15:28,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 082e64a1d02e824a02737c5fd026f5bd, store=B 2024-11-26T19:15:28,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:15:28,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 082e64a1d02e824a02737c5fd026f5bd, store=C 2024-11-26T19:15:28,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:15:28,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112660f333b408d545b7b4c3bb84ee87e821_082e64a1d02e824a02737c5fd026f5bd is 50, key is test_row_0/A:col10/1732648525956/Put/seqid=0 2024-11-26T19:15:28,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742200_1376 (size=12154) 2024-11-26T19:15:28,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(8855): Flush requested on 082e64a1d02e824a02737c5fd026f5bd 2024-11-26T19:15:28,148 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd. as already flushing 2024-11-26T19:15:28,187 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:28,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58914 deadline: 1732648588175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:28,187 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:28,187 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:28,187 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:28,191 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:28,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58842 deadline: 1732648588177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:28,191 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:28,191 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:28,191 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:28,191 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:28,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58878 deadline: 1732648588178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:28,192 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:28,192 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:28,192 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:28,192 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:28,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58858 deadline: 1732648588186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:28,192 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:28,192 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:28,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58864 deadline: 1732648588187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:28,192 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:28,192 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:28,192 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:28,192 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:28,192 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:28,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-26T19:15:28,303 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:28,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58914 deadline: 1732648588291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:28,304 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:28,304 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:28,304 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:28,311 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:28,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58842 deadline: 1732648588301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:28,312 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:28,312 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:28,312 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:28,312 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:28,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58878 deadline: 1732648588301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:28,312 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:28,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58858 deadline: 1732648588302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:28,312 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:28,312 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:28,312 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:28,312 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:28,313 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:28,313 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:28,313 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:28,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58864 deadline: 1732648588302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:28,313 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:28,313 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:28,313 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:28,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:28,493 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112660f333b408d545b7b4c3bb84ee87e821_082e64a1d02e824a02737c5fd026f5bd to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112660f333b408d545b7b4c3bb84ee87e821_082e64a1d02e824a02737c5fd026f5bd 2024-11-26T19:15:28,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp/A/0e2aabc5ec6b4af88397bbc626bc1b7e, store: [table=TestAcidGuarantees family=A region=082e64a1d02e824a02737c5fd026f5bd] 2024-11-26T19:15:28,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp/A/0e2aabc5ec6b4af88397bbc626bc1b7e is 175, key is test_row_0/A:col10/1732648525956/Put/seqid=0 2024-11-26T19:15:28,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742201_1377 (size=30955) 2024-11-26T19:15:28,519 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:28,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58914 deadline: 1732648588512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:28,520 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:28,520 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:28,520 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:28,534 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:28,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58842 deadline: 1732648588522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:28,534 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:28,534 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:28,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58878 deadline: 1732648588522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:28,534 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:28,534 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:28,534 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:28,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58858 deadline: 1732648588522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:28,535 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:28,535 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:28,535 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:28,535 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:28,535 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:28,535 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:28,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58864 deadline: 1732648588522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:28,535 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:28,535 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:28,535 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:28,535 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:28,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-26T19:15:28,837 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:28,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58914 deadline: 1732648588832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:28,837 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:28,837 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:28,837 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:28,850 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:28,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58842 deadline: 1732648588842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:28,851 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:28,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58878 deadline: 1732648588842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:28,851 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:28,851 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:28,851 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:28,851 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:28,851 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:28,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58858 deadline: 1732648588842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:28,851 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:28,851 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:28,851 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:28,851 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:28,851 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:28,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58864 deadline: 1732648588842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:28,851 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:28,851 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:28,852 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:28,852 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:28,899 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=111, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp/A/0e2aabc5ec6b4af88397bbc626bc1b7e 2024-11-26T19:15:28,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp/B/fa6479c9f4014069b8081db22f2fc084 is 50, key is test_row_0/B:col10/1732648525956/Put/seqid=0 2024-11-26T19:15:28,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742202_1378 (size=12001) 2024-11-26T19:15:29,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-26T19:15:29,311 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=111 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp/B/fa6479c9f4014069b8081db22f2fc084 2024-11-26T19:15:29,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp/C/30d8ded948704a30bef1e033058b615a is 50, key is test_row_0/C:col10/1732648525956/Put/seqid=0 2024-11-26T19:15:29,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742203_1379 (size=12001) 2024-11-26T19:15:29,355 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:29,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58914 deadline: 1732648589342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:29,355 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:29,355 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:29,355 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:29,376 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:29,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58842 deadline: 1732648589361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:29,377 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:29,377 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:29,377 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:29,377 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:29,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58878 deadline: 1732648589362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:29,377 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:29,377 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:29,377 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:29,377 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:29,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58858 deadline: 1732648589362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:29,377 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:29,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58864 deadline: 1732648589362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:29,377 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:29,377 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:29,378 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:29,378 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:29,378 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:29,378 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:29,726 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=111 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp/C/30d8ded948704a30bef1e033058b615a 2024-11-26T19:15:29,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp/A/0e2aabc5ec6b4af88397bbc626bc1b7e as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/A/0e2aabc5ec6b4af88397bbc626bc1b7e 2024-11-26T19:15:29,733 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/A/0e2aabc5ec6b4af88397bbc626bc1b7e, entries=150, sequenceid=111, filesize=30.2 K 2024-11-26T19:15:29,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp/B/fa6479c9f4014069b8081db22f2fc084 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/B/fa6479c9f4014069b8081db22f2fc084 2024-11-26T19:15:29,740 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/B/fa6479c9f4014069b8081db22f2fc084, entries=150, sequenceid=111, filesize=11.7 K 2024-11-26T19:15:29,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp/C/30d8ded948704a30bef1e033058b615a as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/C/30d8ded948704a30bef1e033058b615a 2024-11-26T19:15:29,745 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/C/30d8ded948704a30bef1e033058b615a, entries=150, sequenceid=111, filesize=11.7 K 2024-11-26T19:15:29,746 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 082e64a1d02e824a02737c5fd026f5bd in 1679ms, sequenceid=111, compaction requested=true 2024-11-26T19:15:29,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2603): Flush status journal for 082e64a1d02e824a02737c5fd026f5bd: 2024-11-26T19:15:29,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd. 2024-11-26T19:15:29,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=96 2024-11-26T19:15:29,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33811 {}] master.HMaster(4169): Remote procedure done, pid=96 2024-11-26T19:15:29,749 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95 2024-11-26T19:15:29,749 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8320 sec 2024-11-26T19:15:29,750 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees in 1.8370 sec 2024-11-26T19:15:30,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-26T19:15:30,062 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T19:15:30,063 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33811 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T19:15:30,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33811 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees 2024-11-26T19:15:30,065 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T19:15:30,065 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T19:15:30,066 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T19:15:30,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-26T19:15:30,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-26T19:15:30,223 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-26T19:15:30,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd. 2024-11-26T19:15:30,223 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2902): Flushing 082e64a1d02e824a02737c5fd026f5bd 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-26T19:15:30,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 082e64a1d02e824a02737c5fd026f5bd, store=A 2024-11-26T19:15:30,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:15:30,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 082e64a1d02e824a02737c5fd026f5bd, store=B 2024-11-26T19:15:30,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:15:30,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 082e64a1d02e824a02737c5fd026f5bd, store=C 2024-11-26T19:15:30,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:15:30,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126121c78f26c1846c19663c5b1b1b81af8_082e64a1d02e824a02737c5fd026f5bd is 50, key is test_row_0/A:col10/1732648528175/Put/seqid=0 2024-11-26T19:15:30,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742204_1380 (size=12254) 2024-11-26T19:15:30,376 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd. as already flushing 2024-11-26T19:15:30,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(8855): Flush requested on 082e64a1d02e824a02737c5fd026f5bd 2024-11-26T19:15:30,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-26T19:15:30,412 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:30,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58864 deadline: 1732648590402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:30,413 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:30,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58878 deadline: 1732648590403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:30,413 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:30,413 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:30,413 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:30,413 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:30,413 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:30,413 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:30,415 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:30,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58858 deadline: 1732648590403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:30,415 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:30,415 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:30,415 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:30,419 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:30,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58914 deadline: 1732648590406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:30,420 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:30,420 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:30,420 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:30,428 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:30,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58842 deadline: 1732648590413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:30,429 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:30,429 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:30,429 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:30,534 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:30,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58864 deadline: 1732648590522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:30,534 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:30,534 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:30,534 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:30,534 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:30,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58878 deadline: 1732648590522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:30,534 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:30,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58858 deadline: 1732648590522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:30,535 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:30,535 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:30,535 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:30,535 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:30,535 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:30,535 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:30,540 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:30,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58914 deadline: 1732648590532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:30,540 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:30,540 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:30,540 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:30,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58842 deadline: 1732648590532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:30,540 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:30,541 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:30,541 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:30,541 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:30,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:30,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-26T19:15:30,694 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126121c78f26c1846c19663c5b1b1b81af8_082e64a1d02e824a02737c5fd026f5bd to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126121c78f26c1846c19663c5b1b1b81af8_082e64a1d02e824a02737c5fd026f5bd 2024-11-26T19:15:30,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp/A/20a9d62f35e54f7d90060a4137376dbe, store: [table=TestAcidGuarantees family=A region=082e64a1d02e824a02737c5fd026f5bd] 2024-11-26T19:15:30,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp/A/20a9d62f35e54f7d90060a4137376dbe is 175, key is test_row_0/A:col10/1732648528175/Put/seqid=0 2024-11-26T19:15:30,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742205_1381 (size=31055) 2024-11-26T19:15:30,755 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:30,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58864 deadline: 1732648590742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:30,756 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:30,756 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:30,756 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:30,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58878 deadline: 1732648590742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:30,756 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:30,756 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:30,756 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:30,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58858 deadline: 1732648590742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:30,756 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:30,756 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:30,756 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:30,756 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:30,756 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:30,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58914 deadline: 1732648590751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:30,757 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:30,757 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:30,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58842 deadline: 1732648590752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:30,757 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:30,757 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:30,757 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:30,757 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:30,757 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:30,757 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:31,078 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:31,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58878 deadline: 1732648591062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:31,079 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:31,079 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:31,079 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:31,079 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:31,079 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:31,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58914 deadline: 1732648591062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:31,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58858 deadline: 1732648591062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:31,079 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:31,079 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:31,079 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:31,079 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:31,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58864 deadline: 1732648591062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:31,079 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:31,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58842 deadline: 1732648591062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:31,080 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:31,080 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:31,080 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:31,080 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:31,080 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:31,080 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:31,080 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:31,080 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:31,080 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:31,104 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=131, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp/A/20a9d62f35e54f7d90060a4137376dbe 2024-11-26T19:15:31,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp/B/31a96a2a4a8d4fd39028234e2c0896f4 is 50, key is test_row_0/B:col10/1732648528175/Put/seqid=0 2024-11-26T19:15:31,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742206_1382 (size=12101) 2024-11-26T19:15:31,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-26T19:15:31,516 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp/B/31a96a2a4a8d4fd39028234e2c0896f4 2024-11-26T19:15:31,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp/C/210e101a7b34413da00b42b491562021 is 50, key is test_row_0/C:col10/1732648528175/Put/seqid=0 2024-11-26T19:15:31,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742207_1383 (size=12101) 2024-11-26T19:15:31,593 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:31,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58878 deadline: 1732648591582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:31,594 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:31,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58864 deadline: 1732648591582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:31,594 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:31,594 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:31,594 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:31,594 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:31,594 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:31,594 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:31,602 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:31,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58914 deadline: 1732648591592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:31,602 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:31,602 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:31,602 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:31,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58858 deadline: 1732648591592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:31,602 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:31,603 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:31,603 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:31,603 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:31,603 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:31,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58842 deadline: 1732648591592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:31,603 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:31,603 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:31,603 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:31,928 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp/C/210e101a7b34413da00b42b491562021 2024-11-26T19:15:31,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp/A/20a9d62f35e54f7d90060a4137376dbe as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/A/20a9d62f35e54f7d90060a4137376dbe 2024-11-26T19:15:31,936 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/A/20a9d62f35e54f7d90060a4137376dbe, entries=150, sequenceid=131, filesize=30.3 K 2024-11-26T19:15:31,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp/B/31a96a2a4a8d4fd39028234e2c0896f4 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/B/31a96a2a4a8d4fd39028234e2c0896f4 2024-11-26T19:15:31,940 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/B/31a96a2a4a8d4fd39028234e2c0896f4, entries=150, sequenceid=131, filesize=11.8 K 2024-11-26T19:15:31,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp/C/210e101a7b34413da00b42b491562021 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/C/210e101a7b34413da00b42b491562021 2024-11-26T19:15:31,945 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/C/210e101a7b34413da00b42b491562021, entries=150, sequenceid=131, filesize=11.8 K 2024-11-26T19:15:31,946 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 082e64a1d02e824a02737c5fd026f5bd in 1723ms, sequenceid=131, compaction requested=true 2024-11-26T19:15:31,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2603): Flush status journal for 082e64a1d02e824a02737c5fd026f5bd: 2024-11-26T19:15:31,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd. 2024-11-26T19:15:31,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=98 2024-11-26T19:15:31,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33811 {}] master.HMaster(4169): Remote procedure done, pid=98 2024-11-26T19:15:31,949 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-11-26T19:15:31,949 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8810 sec 2024-11-26T19:15:31,950 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees in 1.8860 sec 2024-11-26T19:15:32,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-26T19:15:32,212 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T19:15:32,212 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33811 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T19:15:32,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33811 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees 2024-11-26T19:15:32,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-26T19:15:32,214 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T19:15:32,215 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T19:15:32,215 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T19:15:32,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-26T19:15:32,367 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-26T19:15:32,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd. 2024-11-26T19:15:32,368 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2902): Flushing 082e64a1d02e824a02737c5fd026f5bd 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-26T19:15:32,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 082e64a1d02e824a02737c5fd026f5bd, store=A 2024-11-26T19:15:32,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:15:32,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 082e64a1d02e824a02737c5fd026f5bd, store=B 2024-11-26T19:15:32,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:15:32,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 082e64a1d02e824a02737c5fd026f5bd, store=C 2024-11-26T19:15:32,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:15:32,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126f5ea86887f8149f6920efdbdfa5db7eb_082e64a1d02e824a02737c5fd026f5bd is 50, key is test_row_0/A:col10/1732648530411/Put/seqid=0 2024-11-26T19:15:32,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742208_1384 (size=12304) 2024-11-26T19:15:32,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:32,386 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126f5ea86887f8149f6920efdbdfa5db7eb_082e64a1d02e824a02737c5fd026f5bd to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126f5ea86887f8149f6920efdbdfa5db7eb_082e64a1d02e824a02737c5fd026f5bd 2024-11-26T19:15:32,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp/A/25cd12215aa440659e1a058a9a3b1d98, store: [table=TestAcidGuarantees family=A region=082e64a1d02e824a02737c5fd026f5bd] 2024-11-26T19:15:32,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp/A/25cd12215aa440659e1a058a9a3b1d98 is 175, key is test_row_0/A:col10/1732648530411/Put/seqid=0 2024-11-26T19:15:32,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742209_1385 (size=31105) 2024-11-26T19:15:32,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-26T19:15:32,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(8855): Flush requested on 082e64a1d02e824a02737c5fd026f5bd 2024-11-26T19:15:32,609 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd. as already flushing 2024-11-26T19:15:32,690 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:32,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58878 deadline: 1732648592668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:32,690 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:32,690 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:32,690 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:32,705 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:32,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58858 deadline: 1732648592686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:32,706 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:32,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58864 deadline: 1732648592688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:32,706 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:32,706 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:32,706 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:32,706 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:32,706 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:32,706 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:32,706 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:32,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58842 deadline: 1732648592689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:32,706 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:32,706 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:32,706 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:32,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58914 deadline: 1732648592690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:32,706 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:32,707 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:32,707 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:32,707 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:32,793 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=143, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp/A/25cd12215aa440659e1a058a9a3b1d98 2024-11-26T19:15:32,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp/B/5e6b9b512a3d48ca86c05163bacf8931 is 50, key is test_row_0/B:col10/1732648530411/Put/seqid=0 2024-11-26T19:15:32,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742210_1386 (size=12151) 2024-11-26T19:15:32,816 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:32,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58878 deadline: 1732648592801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:32,817 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:32,817 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:32,817 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:32,818 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:32,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58858 deadline: 1732648592812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:32,819 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:32,819 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:32,819 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:32,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58864 deadline: 1732648592812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:32,819 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:32,819 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:32,819 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:32,819 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:32,819 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:32,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58914 deadline: 1732648592812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:32,819 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:32,819 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:32,819 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:32,819 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:32,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58842 deadline: 1732648592812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:32,820 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:32,820 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:32,820 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:32,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-26T19:15:33,032 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:33,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58878 deadline: 1732648593022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:33,032 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:33,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58858 deadline: 1732648593022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:33,032 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:33,032 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:33,032 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:33,032 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:33,033 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:33,033 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:33,033 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:33,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58864 deadline: 1732648593022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:33,033 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:33,033 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:33,033 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:33,033 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:33,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58842 deadline: 1732648593022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:33,033 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:33,033 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:33,033 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:33,041 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:33,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58914 deadline: 1732648593031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:33,041 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:33,042 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:33,042 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:33,208 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=143 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp/B/5e6b9b512a3d48ca86c05163bacf8931 2024-11-26T19:15:33,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp/C/a252478484474687a535155b03c126b4 is 50, key is test_row_0/C:col10/1732648530411/Put/seqid=0 2024-11-26T19:15:33,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742211_1387 (size=12151) 2024-11-26T19:15:33,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-26T19:15:33,353 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:33,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58878 deadline: 1732648593342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:33,354 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:33,354 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:33,354 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:33,354 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:33,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58858 deadline: 1732648593342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:33,354 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:33,354 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:33,354 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:33,354 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:33,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58864 deadline: 1732648593342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:33,354 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:33,354 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:33,354 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:33,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58842 deadline: 1732648593342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:33,354 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:33,355 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:33,355 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:33,355 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:33,360 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:33,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58914 deadline: 1732648593351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:33,360 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:33,360 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:33,360 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:33,619 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=143 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp/C/a252478484474687a535155b03c126b4 2024-11-26T19:15:33,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp/A/25cd12215aa440659e1a058a9a3b1d98 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/A/25cd12215aa440659e1a058a9a3b1d98 2024-11-26T19:15:33,627 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/A/25cd12215aa440659e1a058a9a3b1d98, entries=150, sequenceid=143, filesize=30.4 K 2024-11-26T19:15:33,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp/B/5e6b9b512a3d48ca86c05163bacf8931 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/B/5e6b9b512a3d48ca86c05163bacf8931 2024-11-26T19:15:33,633 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/B/5e6b9b512a3d48ca86c05163bacf8931, entries=150, sequenceid=143, filesize=11.9 K 2024-11-26T19:15:33,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp/C/a252478484474687a535155b03c126b4 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/C/a252478484474687a535155b03c126b4 2024-11-26T19:15:33,637 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/C/a252478484474687a535155b03c126b4, entries=150, sequenceid=143, filesize=11.9 K 2024-11-26T19:15:33,638 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 082e64a1d02e824a02737c5fd026f5bd in 1270ms, sequenceid=143, compaction requested=true 2024-11-26T19:15:33,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2603): Flush status journal for 082e64a1d02e824a02737c5fd026f5bd: 2024-11-26T19:15:33,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd. 2024-11-26T19:15:33,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-11-26T19:15:33,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33811 {}] master.HMaster(4169): Remote procedure done, pid=100 2024-11-26T19:15:33,641 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-11-26T19:15:33,641 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4240 sec 2024-11-26T19:15:33,643 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees in 1.4300 sec 2024-11-26T19:15:33,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(8855): Flush requested on 082e64a1d02e824a02737c5fd026f5bd 2024-11-26T19:15:33,867 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 082e64a1d02e824a02737c5fd026f5bd 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-26T19:15:33,867 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732648532640, max timestamp=1732648533864Num uniques 859; 2024-11-26T19:15:33,867 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732648532640, max timestamp=1732648533864Num uniques 859; 2024-11-26T19:15:33,868 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-26T19:15:33,868 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-26T19:15:33,868 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732648532640, max timestamp=1732648533864Num uniques 859; 2024-11-26T19:15:33,868 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-26T19:15:33,868 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 082e64a1d02e824a02737c5fd026f5bd, store=A 2024-11-26T19:15:33,868 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T19:15:33,868 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 082e64a1d02e824a02737c5fd026f5bd, store=B 2024-11-26T19:15:33,868 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T19:15:33,868 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 082e64a1d02e824a02737c5fd026f5bd, store=C 2024-11-26T19:15:33,868 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T19:15:33,876 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112699a5b353c63b44adb70b899c9873c715_082e64a1d02e824a02737c5fd026f5bd is 50, key is test_row_0/A:col10/1732648532640/Put/seqid=0 2024-11-26T19:15:33,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742212_1388 (size=17284) 2024-11-26T19:15:33,926 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732648533904, max timestamp=1732648533922Num uniques 859; 2024-11-26T19:15:33,926 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-26T19:15:33,926 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732648533904, max timestamp=1732648533922Num uniques 859; 2024-11-26T19:15:33,926 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-26T19:15:33,926 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732648533904, max timestamp=1732648533922Num uniques 859; 2024-11-26T19:15:33,926 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-26T19:15:33,994 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1732648533965, max timestamp=1732648533989Num uniques 1118; 2024-11-26T19:15:33,994 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1732648533965, max timestamp=1732648533989Num uniques 1118; 2024-11-26T19:15:33,994 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=14558, suffix heap size=144240,new segment heap size=33312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=318 2024-11-26T19:15:33,994 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1732648533965, max timestamp=1732648533989Num uniques 1118; 2024-11-26T19:15:33,994 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=14558, suffix heap size=144240,new segment heap size=33312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=318 2024-11-26T19:15:33,994 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=14558, suffix heap size=144240,new segment heap size=33312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=318 2024-11-26T19:15:34,067 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:34,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58914 deadline: 1732648594042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:34,067 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:34,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58878 deadline: 1732648594043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:34,068 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:34,068 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:34,068 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:34,068 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:34,068 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:34,068 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:34,073 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:34,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58858 deadline: 1732648594059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:34,074 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:34,074 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:34,074 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:34,081 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:34,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58864 deadline: 1732648594061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:34,081 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:34,081 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:34,081 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:34,081 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:34,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58842 deadline: 1732648594067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:34,081 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:34,081 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:34,081 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:34,189 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:34,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58914 deadline: 1732648594172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:34,189 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:34,189 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:34,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58878 deadline: 1732648594172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:34,189 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:34,189 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:34,190 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:34,190 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:34,190 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:34,201 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:34,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58858 deadline: 1732648594182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:34,201 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:34,201 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:34,201 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:34,209 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:34,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58864 deadline: 1732648594192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:34,209 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:34,209 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:34,209 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:34,209 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:34,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58842 deadline: 1732648594192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:34,209 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:34,209 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:34,209 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:34,281 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:34,284 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112699a5b353c63b44adb70b899c9873c715_082e64a1d02e824a02737c5fd026f5bd to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112699a5b353c63b44adb70b899c9873c715_082e64a1d02e824a02737c5fd026f5bd 2024-11-26T19:15:34,285 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp/A/448542bb2a654073b3caff8879439129, store: [table=TestAcidGuarantees family=A region=082e64a1d02e824a02737c5fd026f5bd] 2024-11-26T19:15:34,286 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp/A/448542bb2a654073b3caff8879439129 is 175, key is test_row_0/A:col10/1732648532640/Put/seqid=0 2024-11-26T19:15:34,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742213_1389 (size=48389) 2024-11-26T19:15:34,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-26T19:15:34,362 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T19:15:34,363 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33811 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T19:15:34,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33811 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-11-26T19:15:34,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-26T19:15:34,364 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T19:15:34,365 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T19:15:34,365 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T19:15:34,405 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:34,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58914 deadline: 1732648594392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:34,405 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:34,405 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:34,405 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:34,417 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:34,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58878 deadline: 1732648594401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:34,418 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:34,418 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:34,418 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:34,427 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:34,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58858 deadline: 1732648594412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:34,428 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:34,428 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:34,428 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:34,428 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:34,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58864 deadline: 1732648594412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:34,428 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:34,428 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:34,428 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:34,428 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:34,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58842 deadline: 1732648594412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:34,428 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:34,428 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:34,428 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:34,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-26T19:15:34,518 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-26T19:15:34,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd. 2024-11-26T19:15:34,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd. as already flushing 2024-11-26T19:15:34,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd. 2024-11-26T19:15:34,518 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 082e64a1d02e824a02737c5fd026f5bd, NAME => 'TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:34,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 082e64a1d02e824a02737c5fd026f5bd, NAME => 'TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:34,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 082e64a1d02e824a02737c5fd026f5bd, NAME => 'TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 082e64a1d02e824a02737c5fd026f5bd, NAME => 'TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:34,671 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-26T19:15:34,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd. 2024-11-26T19:15:34,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd. as already flushing 2024-11-26T19:15:34,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd. 2024-11-26T19:15:34,671 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 082e64a1d02e824a02737c5fd026f5bd, NAME => 'TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:34,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 082e64a1d02e824a02737c5fd026f5bd, NAME => 'TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:34,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 082e64a1d02e824a02737c5fd026f5bd, NAME => 'TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 082e64a1d02e824a02737c5fd026f5bd, NAME => 'TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:34,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-26T19:15:34,691 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=165, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp/A/448542bb2a654073b3caff8879439129 2024-11-26T19:15:34,698 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp/B/13460d3ed9b8477d9b5fc92ec379669a is 50, key is test_row_0/B:col10/1732648532640/Put/seqid=0 2024-11-26T19:15:34,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742214_1390 (size=12151) 2024-11-26T19:15:34,721 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:34,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58914 deadline: 1732648594711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:34,722 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:34,722 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:34,722 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:34,739 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:34,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58878 deadline: 1732648594721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:34,739 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:34,739 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:34,739 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:34,740 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:34,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58858 deadline: 1732648594732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:34,741 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:34,741 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:34,741 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:34,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58864 deadline: 1732648594732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:34,741 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:34,741 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:34,741 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:34,741 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:34,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58842 deadline: 1732648594732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:34,741 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:34,742 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:34,742 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:34,742 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:34,824 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-26T19:15:34,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd. 2024-11-26T19:15:34,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd. as already flushing 2024-11-26T19:15:34,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd. 2024-11-26T19:15:34,824 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 082e64a1d02e824a02737c5fd026f5bd, NAME => 'TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:34,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 082e64a1d02e824a02737c5fd026f5bd, NAME => 'TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:34,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 082e64a1d02e824a02737c5fd026f5bd, NAME => 'TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 082e64a1d02e824a02737c5fd026f5bd, NAME => 'TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:34,977 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-26T19:15:34,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd. 2024-11-26T19:15:34,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd. as already flushing 2024-11-26T19:15:34,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd. 2024-11-26T19:15:34,977 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 082e64a1d02e824a02737c5fd026f5bd, NAME => 'TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:34,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 082e64a1d02e824a02737c5fd026f5bd, NAME => 'TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:34,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 082e64a1d02e824a02737c5fd026f5bd, NAME => 'TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 082e64a1d02e824a02737c5fd026f5bd, NAME => 'TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:34,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-26T19:15:35,102 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=165 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp/B/13460d3ed9b8477d9b5fc92ec379669a 2024-11-26T19:15:35,109 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp/C/016e03b18a9e42ef995014296f618a5b is 50, key is test_row_0/C:col10/1732648532640/Put/seqid=0 2024-11-26T19:15:35,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742215_1391 (size=12151) 2024-11-26T19:15:35,130 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-26T19:15:35,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd. 2024-11-26T19:15:35,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd. as already flushing 2024-11-26T19:15:35,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd. 2024-11-26T19:15:35,131 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 082e64a1d02e824a02737c5fd026f5bd, NAME => 'TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:35,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 082e64a1d02e824a02737c5fd026f5bd, NAME => 'TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:35,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 082e64a1d02e824a02737c5fd026f5bd, NAME => 'TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 082e64a1d02e824a02737c5fd026f5bd, NAME => 'TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:35,246 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:35,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58914 deadline: 1732648595232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:35,247 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:35,247 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:35,247 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:35,266 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:35,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58878 deadline: 1732648595252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:35,266 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:35,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58858 deadline: 1732648595252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:35,266 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:35,266 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:35,266 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:35,266 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:35,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58864 deadline: 1732648595252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:35,266 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:35,266 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:35,266 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:35,266 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:35,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58842 deadline: 1732648595252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:35,267 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:35,267 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:35,267 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:35,267 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:35,267 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:35,267 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:35,283 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-26T19:15:35,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd. 2024-11-26T19:15:35,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd. as already flushing 2024-11-26T19:15:35,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd. 2024-11-26T19:15:35,284 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 082e64a1d02e824a02737c5fd026f5bd, NAME => 'TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:35,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 082e64a1d02e824a02737c5fd026f5bd, NAME => 'TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:35,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 082e64a1d02e824a02737c5fd026f5bd, NAME => 'TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 082e64a1d02e824a02737c5fd026f5bd, NAME => 'TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:35,436 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-26T19:15:35,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd. 2024-11-26T19:15:35,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd. as already flushing 2024-11-26T19:15:35,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd. 2024-11-26T19:15:35,437 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 082e64a1d02e824a02737c5fd026f5bd, NAME => 'TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:35,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 082e64a1d02e824a02737c5fd026f5bd, NAME => 'TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:35,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 082e64a1d02e824a02737c5fd026f5bd, NAME => 'TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 082e64a1d02e824a02737c5fd026f5bd, NAME => 'TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:35,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-26T19:15:35,514 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=165 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp/C/016e03b18a9e42ef995014296f618a5b 2024-11-26T19:15:35,517 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp/A/448542bb2a654073b3caff8879439129 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/A/448542bb2a654073b3caff8879439129 2024-11-26T19:15:35,521 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/A/448542bb2a654073b3caff8879439129, entries=250, sequenceid=165, filesize=47.3 K 2024-11-26T19:15:35,521 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp/B/13460d3ed9b8477d9b5fc92ec379669a as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/B/13460d3ed9b8477d9b5fc92ec379669a 2024-11-26T19:15:35,525 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/B/13460d3ed9b8477d9b5fc92ec379669a, entries=150, sequenceid=165, filesize=11.9 K 2024-11-26T19:15:35,526 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp/C/016e03b18a9e42ef995014296f618a5b as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/C/016e03b18a9e42ef995014296f618a5b 2024-11-26T19:15:35,529 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/C/016e03b18a9e42ef995014296f618a5b, entries=150, sequenceid=165, filesize=11.9 K 2024-11-26T19:15:35,530 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~86.11 KB/88176, currentSize=154.31 KB/158010 for 082e64a1d02e824a02737c5fd026f5bd in 1663ms, sequenceid=165, compaction requested=true 2024-11-26T19:15:35,530 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 082e64a1d02e824a02737c5fd026f5bd: 2024-11-26T19:15:35,530 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 082e64a1d02e824a02737c5fd026f5bd:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T19:15:35,530 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:15:35,530 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-26T19:15:35,530 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 082e64a1d02e824a02737c5fd026f5bd:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T19:15:35,530 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:15:35,530 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 082e64a1d02e824a02737c5fd026f5bd:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T19:15:35,530 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T19:15:35,530 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-26T19:15:35,531 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 203517 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-26T19:15:35,531 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 72509 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-26T19:15:35,532 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1541): 082e64a1d02e824a02737c5fd026f5bd/B is initiating minor compaction (all files) 2024-11-26T19:15:35,532 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1541): 082e64a1d02e824a02737c5fd026f5bd/A is initiating minor compaction (all files) 2024-11-26T19:15:35,532 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 082e64a1d02e824a02737c5fd026f5bd/A in TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd. 2024-11-26T19:15:35,532 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 082e64a1d02e824a02737c5fd026f5bd/B in TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd. 2024-11-26T19:15:35,532 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/A/f3ca20aed6e5408abee89210f3ffa1d9, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/A/cec973f41b5643919ab55d6facce6761, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/A/0e2aabc5ec6b4af88397bbc626bc1b7e, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/A/20a9d62f35e54f7d90060a4137376dbe, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/A/25cd12215aa440659e1a058a9a3b1d98, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/A/448542bb2a654073b3caff8879439129] into tmpdir=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp, totalSize=198.7 K 2024-11-26T19:15:35,532 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/B/9e574ca652374eceaed7ac1547ce6564, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/B/699b2dcd81384c4c839aba9062adebee, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/B/fa6479c9f4014069b8081db22f2fc084, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/B/31a96a2a4a8d4fd39028234e2c0896f4, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/B/5e6b9b512a3d48ca86c05163bacf8931, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/B/13460d3ed9b8477d9b5fc92ec379669a] into tmpdir=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp, totalSize=70.8 K 2024-11-26T19:15:35,532 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=10 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd. 2024-11-26T19:15:35,532 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd. files: [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/A/f3ca20aed6e5408abee89210f3ffa1d9, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/A/cec973f41b5643919ab55d6facce6761, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/A/0e2aabc5ec6b4af88397bbc626bc1b7e, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/A/20a9d62f35e54f7d90060a4137376dbe, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/A/25cd12215aa440659e1a058a9a3b1d98, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/A/448542bb2a654073b3caff8879439129] 2024-11-26T19:15:35,532 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting 9e574ca652374eceaed7ac1547ce6564, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732648523860 2024-11-26T19:15:35,532 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting f3ca20aed6e5408abee89210f3ffa1d9, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732648523860 2024-11-26T19:15:35,532 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting 699b2dcd81384c4c839aba9062adebee, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1732648524747 2024-11-26T19:15:35,532 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting cec973f41b5643919ab55d6facce6761, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1732648524747 2024-11-26T19:15:35,533 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting fa6479c9f4014069b8081db22f2fc084, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=111, earliestPutTs=1732648525953 2024-11-26T19:15:35,533 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0e2aabc5ec6b4af88397bbc626bc1b7e, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=111, earliestPutTs=1732648525953 2024-11-26T19:15:35,533 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting 31a96a2a4a8d4fd39028234e2c0896f4, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1732648528175 2024-11-26T19:15:35,533 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting 20a9d62f35e54f7d90060a4137376dbe, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1732648528175 2024-11-26T19:15:35,533 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting 25cd12215aa440659e1a058a9a3b1d98, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1732648530401 2024-11-26T19:15:35,533 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting 5e6b9b512a3d48ca86c05163bacf8931, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1732648530401 2024-11-26T19:15:35,534 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting 448542bb2a654073b3caff8879439129, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1732648532640 2024-11-26T19:15:35,534 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting 13460d3ed9b8477d9b5fc92ec379669a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1732648532640 2024-11-26T19:15:35,554 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 082e64a1d02e824a02737c5fd026f5bd#B#compaction#329 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T19:15:35,554 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp/B/5c1e45e54d5f4766a6ef182c5544fd89 is 50, key is test_row_0/B:col10/1732648532640/Put/seqid=0 2024-11-26T19:15:35,555 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=082e64a1d02e824a02737c5fd026f5bd] 2024-11-26T19:15:35,557 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241126ee56b05854bd4fc0b0050a5640b9216b_082e64a1d02e824a02737c5fd026f5bd store=[table=TestAcidGuarantees family=A region=082e64a1d02e824a02737c5fd026f5bd] 2024-11-26T19:15:35,563 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241126ee56b05854bd4fc0b0050a5640b9216b_082e64a1d02e824a02737c5fd026f5bd, store=[table=TestAcidGuarantees family=A region=082e64a1d02e824a02737c5fd026f5bd] 2024-11-26T19:15:35,563 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126ee56b05854bd4fc0b0050a5640b9216b_082e64a1d02e824a02737c5fd026f5bd because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=082e64a1d02e824a02737c5fd026f5bd] 2024-11-26T19:15:35,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742216_1392 (size=12459) 2024-11-26T19:15:35,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742217_1393 (size=4469) 2024-11-26T19:15:35,589 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-26T19:15:35,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd. 2024-11-26T19:15:35,589 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing 082e64a1d02e824a02737c5fd026f5bd 3/3 column families, dataSize=154.31 KB heapSize=429.84 KB 2024-11-26T19:15:35,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 082e64a1d02e824a02737c5fd026f5bd, store=A 2024-11-26T19:15:35,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T19:15:35,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 082e64a1d02e824a02737c5fd026f5bd, store=B 2024-11-26T19:15:35,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T19:15:35,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 082e64a1d02e824a02737c5fd026f5bd, store=C 2024-11-26T19:15:35,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T19:15:35,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411264b7e60d9caa141c0b25c45807f922f1f_082e64a1d02e824a02737c5fd026f5bd is 50, key is test_row_0/A:col10/1732648534057/Put/seqid=0 2024-11-26T19:15:35,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742218_1394 (size=12304) 2024-11-26T19:15:35,974 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp/B/5c1e45e54d5f4766a6ef182c5544fd89 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/B/5c1e45e54d5f4766a6ef182c5544fd89 2024-11-26T19:15:35,979 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 082e64a1d02e824a02737c5fd026f5bd#A#compaction#330 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T19:15:35,979 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in 082e64a1d02e824a02737c5fd026f5bd/B of 082e64a1d02e824a02737c5fd026f5bd into 5c1e45e54d5f4766a6ef182c5544fd89(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T19:15:35,979 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 082e64a1d02e824a02737c5fd026f5bd: 2024-11-26T19:15:35,979 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., storeName=082e64a1d02e824a02737c5fd026f5bd/B, priority=10, startTime=1732648535530; duration=0sec 2024-11-26T19:15:35,979 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp/A/e0f8f052966a4c3da4e38cc3d9be7da0 is 175, key is test_row_0/A:col10/1732648532640/Put/seqid=0 2024-11-26T19:15:35,979 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T19:15:35,979 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 082e64a1d02e824a02737c5fd026f5bd:B 2024-11-26T19:15:35,979 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-26T19:15:35,981 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 72509 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-26T19:15:35,981 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1541): 082e64a1d02e824a02737c5fd026f5bd/C is initiating minor compaction (all files) 2024-11-26T19:15:35,981 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 082e64a1d02e824a02737c5fd026f5bd/C in TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd. 2024-11-26T19:15:35,981 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/C/2d9fbfcec4ef4a38b1a508e06f99281e, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/C/4c85c64e413b422197212ce46d79ad66, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/C/30d8ded948704a30bef1e033058b615a, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/C/210e101a7b34413da00b42b491562021, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/C/a252478484474687a535155b03c126b4, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/C/016e03b18a9e42ef995014296f618a5b] into tmpdir=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp, totalSize=70.8 K 2024-11-26T19:15:35,982 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting 2d9fbfcec4ef4a38b1a508e06f99281e, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732648523860 2024-11-26T19:15:35,982 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting 4c85c64e413b422197212ce46d79ad66, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1732648524747 2024-11-26T19:15:35,982 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting 30d8ded948704a30bef1e033058b615a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=111, earliestPutTs=1732648525953 2024-11-26T19:15:35,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742219_1395 (size=31413) 2024-11-26T19:15:35,983 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting 210e101a7b34413da00b42b491562021, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1732648528175 2024-11-26T19:15:35,983 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting a252478484474687a535155b03c126b4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1732648530401 2024-11-26T19:15:35,983 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting 016e03b18a9e42ef995014296f618a5b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1732648532640 2024-11-26T19:15:35,993 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 082e64a1d02e824a02737c5fd026f5bd#C#compaction#332 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T19:15:35,993 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp/C/a391bd18f1fd4a65a1c74049bf382942 is 50, key is test_row_0/C:col10/1732648532640/Put/seqid=0 2024-11-26T19:15:36,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,009 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411264b7e60d9caa141c0b25c45807f922f1f_082e64a1d02e824a02737c5fd026f5bd to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411264b7e60d9caa141c0b25c45807f922f1f_082e64a1d02e824a02737c5fd026f5bd 2024-11-26T19:15:36,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742220_1396 (size=12459) 2024-11-26T19:15:36,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp/A/cd689d7aad9e434e968235e59a87452e, store: [table=TestAcidGuarantees family=A region=082e64a1d02e824a02737c5fd026f5bd] 2024-11-26T19:15:36,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp/A/cd689d7aad9e434e968235e59a87452e is 175, key is test_row_0/A:col10/1732648534057/Put/seqid=0 2024-11-26T19:15:36,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742221_1397 (size=31105) 2024-11-26T19:15:36,018 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=219, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp/A/cd689d7aad9e434e968235e59a87452e 2024-11-26T19:15:36,018 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp/C/a391bd18f1fd4a65a1c74049bf382942 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/C/a391bd18f1fd4a65a1c74049bf382942 2024-11-26T19:15:36,026 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in 082e64a1d02e824a02737c5fd026f5bd/C of 082e64a1d02e824a02737c5fd026f5bd into a391bd18f1fd4a65a1c74049bf382942(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T19:15:36,026 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 082e64a1d02e824a02737c5fd026f5bd: 2024-11-26T19:15:36,026 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., storeName=082e64a1d02e824a02737c5fd026f5bd/C, priority=10, startTime=1732648535530; duration=0sec 2024-11-26T19:15:36,026 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:15:36,027 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 082e64a1d02e824a02737c5fd026f5bd:C 2024-11-26T19:15:36,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp/B/53c11c5b02fb44b99512e8521712f7e0 is 50, key is test_row_0/B:col10/1732648534057/Put/seqid=0 2024-11-26T19:15:36,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742222_1398 (size=12151) 2024-11-26T19:15:36,035 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=219 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp/B/53c11c5b02fb44b99512e8521712f7e0 2024-11-26T19:15:36,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp/C/9cd31dffa1c84f18a869b2e579c993b4 is 50, key is test_row_0/C:col10/1732648534057/Put/seqid=0 2024-11-26T19:15:36,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742223_1399 (size=12151) 2024-11-26T19:15:36,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(8855): Flush requested on 082e64a1d02e824a02737c5fd026f5bd 2024-11-26T19:15:36,273 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd. as already flushing 2024-11-26T19:15:36,295 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:36,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58864 deadline: 1732648596282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:36,296 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:36,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58842 deadline: 1732648596282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:36,296 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to fe76acf732b2,42785,1732648423970 for 'test_row_1' in 082e64a1d02e824a02737c5fd026f5bd of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2235 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T19:15:36,296 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to fe76acf732b2,42785,1732648423970 for 'test_row_1' in 082e64a1d02e824a02737c5fd026f5bd of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2229 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T19:15:36,297 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:36,297 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:36,297 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:36,297 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:36,297 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:36,297 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:36,298 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:36,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58914 deadline: 1732648596291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:36,298 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:36,298 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:36,298 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:36,305 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:36,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58878 deadline: 1732648596294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:36,305 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:36,305 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:36,305 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:36,311 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:36,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58858 deadline: 1732648596296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:36,312 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:36,312 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:36,312 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:36,387 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp/A/e0f8f052966a4c3da4e38cc3d9be7da0 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/A/e0f8f052966a4c3da4e38cc3d9be7da0 2024-11-26T19:15:36,392 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in 082e64a1d02e824a02737c5fd026f5bd/A of 082e64a1d02e824a02737c5fd026f5bd into e0f8f052966a4c3da4e38cc3d9be7da0(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T19:15:36,392 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 082e64a1d02e824a02737c5fd026f5bd: 2024-11-26T19:15:36,392 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., storeName=082e64a1d02e824a02737c5fd026f5bd/A, priority=10, startTime=1732648535530; duration=0sec 2024-11-26T19:15:36,392 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:15:36,392 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 082e64a1d02e824a02737c5fd026f5bd:A 2024-11-26T19:15:36,417 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:36,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58914 deadline: 1732648596402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:36,417 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:36,417 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:36,417 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:36,419 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:36,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58878 deadline: 1732648596412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:36,419 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:36,420 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:36,420 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:36,434 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:36,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58858 deadline: 1732648596421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:36,434 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:36,434 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:36,434 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:36,455 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=219 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp/C/9cd31dffa1c84f18a869b2e579c993b4 2024-11-26T19:15:36,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp/A/cd689d7aad9e434e968235e59a87452e as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/A/cd689d7aad9e434e968235e59a87452e 2024-11-26T19:15:36,465 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/A/cd689d7aad9e434e968235e59a87452e, entries=150, sequenceid=219, filesize=30.4 K 2024-11-26T19:15:36,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp/B/53c11c5b02fb44b99512e8521712f7e0 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/B/53c11c5b02fb44b99512e8521712f7e0 2024-11-26T19:15:36,469 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/B/53c11c5b02fb44b99512e8521712f7e0, entries=150, sequenceid=219, filesize=11.9 K 2024-11-26T19:15:36,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp/C/9cd31dffa1c84f18a869b2e579c993b4 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/C/9cd31dffa1c84f18a869b2e579c993b4 2024-11-26T19:15:36,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,474 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/C/9cd31dffa1c84f18a869b2e579c993b4, entries=150, sequenceid=219, filesize=11.9 K 2024-11-26T19:15:36,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,475 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~429.80 KB/440112, currentSize=33.54 KB/34350 for 082e64a1d02e824a02737c5fd026f5bd in 886ms, sequenceid=219, compaction requested=false 2024-11-26T19:15:36,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for 082e64a1d02e824a02737c5fd026f5bd: 2024-11-26T19:15:36,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd. 2024-11-26T19:15:36,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-11-26T19:15:36,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33811 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-11-26T19:15:36,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,479 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-11-26T19:15:36,479 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1120 sec 2024-11-26T19:15:36,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,480 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 2.1160 sec 2024-11-26T19:15:36,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-26T19:15:36,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,512 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T19:15:36,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,512 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33811 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T19:15:36,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33811 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-11-26T19:15:36,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-26T19:15:36,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,514 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T19:15:36,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,515 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T19:15:36,515 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T19:15:36,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-26T19:15:36,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(8855): Flush requested on 082e64a1d02e824a02737c5fd026f5bd 2024-11-26T19:15:36,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,658 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 082e64a1d02e824a02737c5fd026f5bd 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-26T19:15:36,658 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 082e64a1d02e824a02737c5fd026f5bd, store=A 2024-11-26T19:15:36,658 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:15:36,658 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 082e64a1d02e824a02737c5fd026f5bd, store=B 2024-11-26T19:15:36,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:15:36,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 082e64a1d02e824a02737c5fd026f5bd, store=C 2024-11-26T19:15:36,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:15:36,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,666 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112673279be3234e4c059b326c0360f2f9cd_082e64a1d02e824a02737c5fd026f5bd is 50, key is test_row_0/A:col10/1732648536656/Put/seqid=0 2024-11-26T19:15:36,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,669 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-26T19:15:36,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd. 2024-11-26T19:15:36,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd. as already flushing 2024-11-26T19:15:36,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd. 2024-11-26T19:15:36,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,669 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 082e64a1d02e824a02737c5fd026f5bd, NAME => 'TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:36,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 082e64a1d02e824a02737c5fd026f5bd, NAME => 'TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:36,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 082e64a1d02e824a02737c5fd026f5bd, NAME => 'TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 082e64a1d02e824a02737c5fd026f5bd, NAME => 'TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:36,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742224_1400 (size=14794) 2024-11-26T19:15:36,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,683 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,687 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112673279be3234e4c059b326c0360f2f9cd_082e64a1d02e824a02737c5fd026f5bd to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112673279be3234e4c059b326c0360f2f9cd_082e64a1d02e824a02737c5fd026f5bd 2024-11-26T19:15:36,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,688 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp/A/47f3b7df4b984c63972851f6beeae636, store: [table=TestAcidGuarantees family=A region=082e64a1d02e824a02737c5fd026f5bd] 2024-11-26T19:15:36,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,689 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp/A/47f3b7df4b984c63972851f6beeae636 is 175, key is test_row_0/A:col10/1732648536656/Put/seqid=0 2024-11-26T19:15:36,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:36,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742225_1401 (size=39745) 2024-11-26T19:15:36,764 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732648536727, max timestamp=1732648536760Num uniques 859; 2024-11-26T19:15:36,764 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732648536727, max timestamp=1732648536760Num uniques 859; 2024-11-26T19:15:36,764 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-26T19:15:36,764 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-26T19:15:36,764 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732648536727, max timestamp=1732648536760Num uniques 859; 2024-11-26T19:15:36,764 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-26T19:15:36,822 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-26T19:15:36,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd. 2024-11-26T19:15:36,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd. as already flushing 2024-11-26T19:15:36,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd. 2024-11-26T19:15:36,823 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 082e64a1d02e824a02737c5fd026f5bd, NAME => 'TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:36,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 082e64a1d02e824a02737c5fd026f5bd, NAME => 'TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:36,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 082e64a1d02e824a02737c5fd026f5bd, NAME => 'TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 082e64a1d02e824a02737c5fd026f5bd, NAME => 'TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:36,830 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:36,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58858 deadline: 1732648596827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:36,830 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:36,830 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:36,830 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:36,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-26T19:15:36,841 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:36,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58878 deadline: 1732648596829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:36,842 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:36,842 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:36,842 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:36,843 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:36,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58914 deadline: 1732648596830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:36,844 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:36,844 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:36,844 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:36,942 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:36,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58858 deadline: 1732648596931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:36,943 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:36,943 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:36,943 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:36,963 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:36,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58878 deadline: 1732648596952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:36,963 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:36,963 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:36,963 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:36,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58914 deadline: 1732648596952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:36,963 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:36,964 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:36,964 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:36,964 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:36,975 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-26T19:15:36,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd. 2024-11-26T19:15:36,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd. as already flushing 2024-11-26T19:15:36,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd. 2024-11-26T19:15:36,976 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 082e64a1d02e824a02737c5fd026f5bd, NAME => 'TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:36,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 082e64a1d02e824a02737c5fd026f5bd, NAME => 'TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:36,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 082e64a1d02e824a02737c5fd026f5bd, NAME => 'TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 082e64a1d02e824a02737c5fd026f5bd, NAME => 'TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:37,102 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=232, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp/A/47f3b7df4b984c63972851f6beeae636 2024-11-26T19:15:37,110 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp/B/9e41b20c2437410fa6d837307c720f05 is 50, key is test_row_0/B:col10/1732648536656/Put/seqid=0 2024-11-26T19:15:37,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742226_1402 (size=9757) 2024-11-26T19:15:37,128 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-26T19:15:37,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd. 2024-11-26T19:15:37,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd. as already flushing 2024-11-26T19:15:37,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd. 2024-11-26T19:15:37,129 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 082e64a1d02e824a02737c5fd026f5bd, NAME => 'TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:37,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 082e64a1d02e824a02737c5fd026f5bd, NAME => 'TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:37,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 082e64a1d02e824a02737c5fd026f5bd, NAME => 'TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 082e64a1d02e824a02737c5fd026f5bd, NAME => 'TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:37,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-26T19:15:37,157 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:37,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58858 deadline: 1732648597152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:37,157 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:37,157 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:37,157 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:37,184 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:37,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58878 deadline: 1732648597172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:37,184 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:37,184 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:37,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58914 deadline: 1732648597172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:37,184 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:37,184 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:37,184 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:37,184 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:37,184 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:37,281 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-26T19:15:37,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd. 2024-11-26T19:15:37,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd. as already flushing 2024-11-26T19:15:37,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd. 2024-11-26T19:15:37,282 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 082e64a1d02e824a02737c5fd026f5bd, NAME => 'TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:37,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 082e64a1d02e824a02737c5fd026f5bd, NAME => 'TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:37,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 082e64a1d02e824a02737c5fd026f5bd, NAME => 'TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 082e64a1d02e824a02737c5fd026f5bd, NAME => 'TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:37,434 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-26T19:15:37,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd. 2024-11-26T19:15:37,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd. as already flushing 2024-11-26T19:15:37,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd. 2024-11-26T19:15:37,435 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 082e64a1d02e824a02737c5fd026f5bd, NAME => 'TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:37,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 082e64a1d02e824a02737c5fd026f5bd, NAME => 'TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:37,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 082e64a1d02e824a02737c5fd026f5bd, NAME => 'TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 082e64a1d02e824a02737c5fd026f5bd, NAME => 'TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:37,473 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:37,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58858 deadline: 1732648597461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:37,473 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:37,474 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:37,474 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:37,502 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:37,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58878 deadline: 1732648597492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:37,502 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:37,503 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:37,503 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:37,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58914 deadline: 1732648597492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:37,503 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:37,503 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:37,503 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:37,503 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:37,517 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=232 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp/B/9e41b20c2437410fa6d837307c720f05 2024-11-26T19:15:37,524 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp/C/27b6b7f7acf949baaed66b10b55ba7d7 is 50, key is test_row_0/C:col10/1732648536656/Put/seqid=0 2024-11-26T19:15:37,528 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742227_1403 (size=9757) 2024-11-26T19:15:37,528 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=232 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp/C/27b6b7f7acf949baaed66b10b55ba7d7 2024-11-26T19:15:37,533 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp/A/47f3b7df4b984c63972851f6beeae636 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/A/47f3b7df4b984c63972851f6beeae636 2024-11-26T19:15:37,536 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/A/47f3b7df4b984c63972851f6beeae636, entries=200, sequenceid=232, filesize=38.8 K 2024-11-26T19:15:37,537 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp/B/9e41b20c2437410fa6d837307c720f05 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/B/9e41b20c2437410fa6d837307c720f05 2024-11-26T19:15:37,541 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/B/9e41b20c2437410fa6d837307c720f05, entries=100, sequenceid=232, filesize=9.5 K 2024-11-26T19:15:37,541 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp/C/27b6b7f7acf949baaed66b10b55ba7d7 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/C/27b6b7f7acf949baaed66b10b55ba7d7 2024-11-26T19:15:37,545 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/C/27b6b7f7acf949baaed66b10b55ba7d7, entries=100, sequenceid=232, filesize=9.5 K 2024-11-26T19:15:37,546 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 082e64a1d02e824a02737c5fd026f5bd in 887ms, sequenceid=232, compaction requested=true 2024-11-26T19:15:37,546 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 082e64a1d02e824a02737c5fd026f5bd: 2024-11-26T19:15:37,546 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 082e64a1d02e824a02737c5fd026f5bd:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T19:15:37,546 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:15:37,546 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T19:15:37,546 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 082e64a1d02e824a02737c5fd026f5bd:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T19:15:37,546 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:15:37,546 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 082e64a1d02e824a02737c5fd026f5bd:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T19:15:37,546 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T19:15:37,546 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T19:15:37,547 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34367 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T19:15:37,547 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102263 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T19:15:37,547 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1541): 082e64a1d02e824a02737c5fd026f5bd/B is initiating minor compaction (all files) 2024-11-26T19:15:37,547 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1541): 082e64a1d02e824a02737c5fd026f5bd/A is initiating minor compaction (all files) 2024-11-26T19:15:37,547 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 082e64a1d02e824a02737c5fd026f5bd/B in TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd. 2024-11-26T19:15:37,547 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 082e64a1d02e824a02737c5fd026f5bd/A in TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd. 2024-11-26T19:15:37,547 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/B/5c1e45e54d5f4766a6ef182c5544fd89, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/B/53c11c5b02fb44b99512e8521712f7e0, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/B/9e41b20c2437410fa6d837307c720f05] into tmpdir=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp, totalSize=33.6 K 2024-11-26T19:15:37,547 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/A/e0f8f052966a4c3da4e38cc3d9be7da0, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/A/cd689d7aad9e434e968235e59a87452e, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/A/47f3b7df4b984c63972851f6beeae636] into tmpdir=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp, totalSize=99.9 K 2024-11-26T19:15:37,547 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd. 2024-11-26T19:15:37,547 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd. files: [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/A/e0f8f052966a4c3da4e38cc3d9be7da0, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/A/cd689d7aad9e434e968235e59a87452e, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/A/47f3b7df4b984c63972851f6beeae636] 2024-11-26T19:15:37,547 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting 5c1e45e54d5f4766a6ef182c5544fd89, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1732648532640 2024-11-26T19:15:37,547 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting e0f8f052966a4c3da4e38cc3d9be7da0, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1732648532640 2024-11-26T19:15:37,548 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting 53c11c5b02fb44b99512e8521712f7e0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=219, earliestPutTs=1732648534057 2024-11-26T19:15:37,548 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting cd689d7aad9e434e968235e59a87452e, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=219, earliestPutTs=1732648534057 2024-11-26T19:15:37,548 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting 9e41b20c2437410fa6d837307c720f05, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1732648536293 2024-11-26T19:15:37,548 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting 47f3b7df4b984c63972851f6beeae636, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1732648536293 2024-11-26T19:15:37,554 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=082e64a1d02e824a02737c5fd026f5bd] 2024-11-26T19:15:37,557 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 082e64a1d02e824a02737c5fd026f5bd#B#compaction#339 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T19:15:37,558 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp/B/333d6b85db774a4ebaedc1fdac0fbc6b is 50, key is test_row_0/B:col10/1732648536656/Put/seqid=0 2024-11-26T19:15:37,560 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241126890aebc4d19944b1a2d0923bd8f50158_082e64a1d02e824a02737c5fd026f5bd store=[table=TestAcidGuarantees family=A region=082e64a1d02e824a02737c5fd026f5bd] 2024-11-26T19:15:37,562 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241126890aebc4d19944b1a2d0923bd8f50158_082e64a1d02e824a02737c5fd026f5bd, store=[table=TestAcidGuarantees family=A region=082e64a1d02e824a02737c5fd026f5bd] 2024-11-26T19:15:37,562 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126890aebc4d19944b1a2d0923bd8f50158_082e64a1d02e824a02737c5fd026f5bd because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=082e64a1d02e824a02737c5fd026f5bd] 2024-11-26T19:15:37,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742228_1404 (size=12561) 2024-11-26T19:15:37,574 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp/B/333d6b85db774a4ebaedc1fdac0fbc6b as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/B/333d6b85db774a4ebaedc1fdac0fbc6b 2024-11-26T19:15:37,578 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 082e64a1d02e824a02737c5fd026f5bd/B of 082e64a1d02e824a02737c5fd026f5bd into 333d6b85db774a4ebaedc1fdac0fbc6b(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T19:15:37,578 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 082e64a1d02e824a02737c5fd026f5bd: 2024-11-26T19:15:37,578 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., storeName=082e64a1d02e824a02737c5fd026f5bd/B, priority=13, startTime=1732648537546; duration=0sec 2024-11-26T19:15:37,578 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T19:15:37,578 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 082e64a1d02e824a02737c5fd026f5bd:B 2024-11-26T19:15:37,578 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T19:15:37,579 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34367 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T19:15:37,579 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1541): 082e64a1d02e824a02737c5fd026f5bd/C is initiating minor compaction (all files) 2024-11-26T19:15:37,579 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 082e64a1d02e824a02737c5fd026f5bd/C in TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd. 2024-11-26T19:15:37,579 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/C/a391bd18f1fd4a65a1c74049bf382942, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/C/9cd31dffa1c84f18a869b2e579c993b4, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/C/27b6b7f7acf949baaed66b10b55ba7d7] into tmpdir=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp, totalSize=33.6 K 2024-11-26T19:15:37,580 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting a391bd18f1fd4a65a1c74049bf382942, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1732648532640 2024-11-26T19:15:37,580 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting 9cd31dffa1c84f18a869b2e579c993b4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=219, earliestPutTs=1732648534057 2024-11-26T19:15:37,580 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting 27b6b7f7acf949baaed66b10b55ba7d7, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1732648536293 2024-11-26T19:15:37,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742229_1405 (size=4469) 2024-11-26T19:15:37,582 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 082e64a1d02e824a02737c5fd026f5bd#A#compaction#338 average throughput is 0.87 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T19:15:37,583 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp/A/cfb1a3a055d84b64bad2b85692d1b5aa is 175, key is test_row_0/A:col10/1732648536656/Put/seqid=0 2024-11-26T19:15:37,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742230_1406 (size=31622) 2024-11-26T19:15:37,587 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-26T19:15:37,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd. 2024-11-26T19:15:37,587 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2902): Flushing 082e64a1d02e824a02737c5fd026f5bd 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-26T19:15:37,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 082e64a1d02e824a02737c5fd026f5bd, store=A 2024-11-26T19:15:37,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T19:15:37,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 082e64a1d02e824a02737c5fd026f5bd, store=B 2024-11-26T19:15:37,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T19:15:37,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 082e64a1d02e824a02737c5fd026f5bd, store=C 2024-11-26T19:15:37,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T19:15:37,590 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 082e64a1d02e824a02737c5fd026f5bd#C#compaction#340 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T19:15:37,591 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp/C/d54e9ff5b6b4458487776e505f9c95e7 is 50, key is test_row_0/C:col10/1732648536656/Put/seqid=0 2024-11-26T19:15:37,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411267daac7e2e2364f11b220af20e7a20eb3_082e64a1d02e824a02737c5fd026f5bd is 50, key is test_row_0/A:col10/1732648536827/Put/seqid=0 2024-11-26T19:15:37,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742232_1408 (size=12454) 2024-11-26T19:15:37,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742231_1407 (size=12561) 2024-11-26T19:15:37,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-26T19:15:37,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(8855): Flush requested on 082e64a1d02e824a02737c5fd026f5bd 2024-11-26T19:15:37,984 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd. as already flushing 2024-11-26T19:15:37,990 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp/A/cfb1a3a055d84b64bad2b85692d1b5aa as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/A/cfb1a3a055d84b64bad2b85692d1b5aa 2024-11-26T19:15:37,994 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 082e64a1d02e824a02737c5fd026f5bd/A of 082e64a1d02e824a02737c5fd026f5bd into cfb1a3a055d84b64bad2b85692d1b5aa(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T19:15:37,994 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 082e64a1d02e824a02737c5fd026f5bd: 2024-11-26T19:15:37,995 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., storeName=082e64a1d02e824a02737c5fd026f5bd/A, priority=13, startTime=1732648537546; duration=0sec 2024-11-26T19:15:37,995 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:15:37,995 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 082e64a1d02e824a02737c5fd026f5bd:A 2024-11-26T19:15:37,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:38,003 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411267daac7e2e2364f11b220af20e7a20eb3_082e64a1d02e824a02737c5fd026f5bd to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411267daac7e2e2364f11b220af20e7a20eb3_082e64a1d02e824a02737c5fd026f5bd 2024-11-26T19:15:38,003 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp/C/d54e9ff5b6b4458487776e505f9c95e7 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/C/d54e9ff5b6b4458487776e505f9c95e7 2024-11-26T19:15:38,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp/A/a48e5f57dfe74028a060103e0b8ead39, store: [table=TestAcidGuarantees family=A region=082e64a1d02e824a02737c5fd026f5bd] 2024-11-26T19:15:38,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp/A/a48e5f57dfe74028a060103e0b8ead39 is 175, key is test_row_0/A:col10/1732648536827/Put/seqid=0 2024-11-26T19:15:38,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742233_1409 (size=31255) 2024-11-26T19:15:38,016 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 082e64a1d02e824a02737c5fd026f5bd/C of 082e64a1d02e824a02737c5fd026f5bd into d54e9ff5b6b4458487776e505f9c95e7(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T19:15:38,016 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 082e64a1d02e824a02737c5fd026f5bd: 2024-11-26T19:15:38,016 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., storeName=082e64a1d02e824a02737c5fd026f5bd/C, priority=13, startTime=1732648537546; duration=0sec 2024-11-26T19:15:38,016 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:15:38,016 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 082e64a1d02e824a02737c5fd026f5bd:C 2024-11-26T19:15:38,043 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:38,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58858 deadline: 1732648598031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:38,044 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:38,044 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:38,044 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:38,045 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:38,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58878 deadline: 1732648598044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:38,045 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:38,045 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:38,045 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:38,045 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:38,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58914 deadline: 1732648598044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:38,046 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:38,046 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:38,046 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:38,160 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:38,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58858 deadline: 1732648598151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:38,161 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:38,161 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:38,161 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:38,161 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:38,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58878 deadline: 1732648598151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:38,161 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:38,161 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:38,161 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:38,161 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:38,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58914 deadline: 1732648598152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:38,161 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:38,161 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:38,161 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:38,323 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:38,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58842 deadline: 1732648598312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:38,323 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to fe76acf732b2,42785,1732648423970 for 'test_row_1' in 082e64a1d02e824a02737c5fd026f5bd of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4256 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T19:15:38,324 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:38,324 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:38,324 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:38,334 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:38,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58864 deadline: 1732648598322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:38,334 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to fe76acf732b2,42785,1732648423970 for 'test_row_1' in 082e64a1d02e824a02737c5fd026f5bd of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4273 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T19:15:38,334 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:38,334 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:38,335 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:38,380 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:38,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58858 deadline: 1732648598371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:38,380 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:38,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58878 deadline: 1732648598372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:38,380 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:38,380 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:38,380 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:38,380 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:38,380 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:38,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58914 deadline: 1732648598372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:38,380 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:38,380 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:38,381 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:38,381 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:38,381 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:38,409 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=269, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp/A/a48e5f57dfe74028a060103e0b8ead39 2024-11-26T19:15:38,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp/B/76d1e48a0b9e4f03aa58b0944b0f9bf3 is 50, key is test_row_0/B:col10/1732648536827/Put/seqid=0 2024-11-26T19:15:38,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742234_1410 (size=12301) 2024-11-26T19:15:38,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-26T19:15:38,698 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:38,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58858 deadline: 1732648598691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:38,698 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:38,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58878 deadline: 1732648598691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:38,698 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:38,698 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:38,698 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:38,698 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:38,699 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:38,699 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:38,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58914 deadline: 1732648598692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:38,699 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:38,699 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:38,699 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:38,699 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:38,823 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=269 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp/B/76d1e48a0b9e4f03aa58b0944b0f9bf3 2024-11-26T19:15:38,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp/C/80151d68b0b44e5fafb6fb2a77472eb3 is 50, key is test_row_0/C:col10/1732648536827/Put/seqid=0 2024-11-26T19:15:38,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742235_1411 (size=12301) 2024-11-26T19:15:39,205 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:39,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58858 deadline: 1732648599202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:39,205 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:39,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58878 deadline: 1732648599202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:39,205 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:39,205 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:39,205 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:39,205 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:39,205 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:39,205 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:39,220 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:39,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58914 deadline: 1732648599212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:39,221 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:39,221 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:39,221 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:39,234 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=269 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp/C/80151d68b0b44e5fafb6fb2a77472eb3 2024-11-26T19:15:39,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp/A/a48e5f57dfe74028a060103e0b8ead39 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/A/a48e5f57dfe74028a060103e0b8ead39 2024-11-26T19:15:39,243 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/A/a48e5f57dfe74028a060103e0b8ead39, entries=150, sequenceid=269, filesize=30.5 K 2024-11-26T19:15:39,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp/B/76d1e48a0b9e4f03aa58b0944b0f9bf3 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/B/76d1e48a0b9e4f03aa58b0944b0f9bf3 2024-11-26T19:15:39,247 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/B/76d1e48a0b9e4f03aa58b0944b0f9bf3, entries=150, sequenceid=269, filesize=12.0 K 2024-11-26T19:15:39,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp/C/80151d68b0b44e5fafb6fb2a77472eb3 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/C/80151d68b0b44e5fafb6fb2a77472eb3 2024-11-26T19:15:39,252 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/C/80151d68b0b44e5fafb6fb2a77472eb3, entries=150, sequenceid=269, filesize=12.0 K 2024-11-26T19:15:39,253 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=53.67 KB/54960 for 082e64a1d02e824a02737c5fd026f5bd in 1665ms, sequenceid=269, compaction requested=false 2024-11-26T19:15:39,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2603): Flush status journal for 082e64a1d02e824a02737c5fd026f5bd: 2024-11-26T19:15:39,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd. 2024-11-26T19:15:39,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=104 2024-11-26T19:15:39,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33811 {}] master.HMaster(4169): Remote procedure done, pid=104 2024-11-26T19:15:39,256 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-11-26T19:15:39,256 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7390 sec 2024-11-26T19:15:39,257 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 2.7440 sec 2024-11-26T19:15:40,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(8855): Flush requested on 082e64a1d02e824a02737c5fd026f5bd 2024-11-26T19:15:40,227 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 082e64a1d02e824a02737c5fd026f5bd 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-26T19:15:40,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 082e64a1d02e824a02737c5fd026f5bd, store=A 2024-11-26T19:15:40,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:15:40,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 082e64a1d02e824a02737c5fd026f5bd, store=B 2024-11-26T19:15:40,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:15:40,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 082e64a1d02e824a02737c5fd026f5bd, store=C 2024-11-26T19:15:40,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:15:40,234 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126420f5150aa9145689cc39cc41515cb16_082e64a1d02e824a02737c5fd026f5bd is 50, key is test_row_0/A:col10/1732648538042/Put/seqid=0 2024-11-26T19:15:40,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742236_1412 (size=17534) 2024-11-26T19:15:40,294 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:40,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58878 deadline: 1732648600284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:40,295 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:40,295 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:40,295 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:40,297 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:40,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58914 deadline: 1732648600292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:40,298 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:40,298 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:40,298 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:40,306 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:40,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58858 deadline: 1732648600293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:40,307 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:40,307 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:40,307 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:40,417 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:40,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58878 deadline: 1732648600403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:40,417 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:40,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58914 deadline: 1732648600403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:40,417 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:40,418 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:40,418 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:40,418 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:40,418 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:40,418 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:40,418 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:40,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58858 deadline: 1732648600414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:40,419 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:40,419 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:40,419 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:40,633 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:40,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58878 deadline: 1732648600621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:40,634 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:40,634 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:40,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58914 deadline: 1732648600622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:40,634 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:40,634 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:40,634 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:40,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58858 deadline: 1732648600622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:40,634 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:40,634 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:40,634 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:40,634 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:40,635 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:40,635 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:40,640 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:40,644 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126420f5150aa9145689cc39cc41515cb16_082e64a1d02e824a02737c5fd026f5bd to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126420f5150aa9145689cc39cc41515cb16_082e64a1d02e824a02737c5fd026f5bd 2024-11-26T19:15:40,645 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp/A/b009e4aade0c410889ba48b66f438038, store: [table=TestAcidGuarantees family=A region=082e64a1d02e824a02737c5fd026f5bd] 2024-11-26T19:15:40,645 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp/A/b009e4aade0c410889ba48b66f438038 is 175, key is test_row_0/A:col10/1732648538042/Put/seqid=0 2024-11-26T19:15:40,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742237_1413 (size=48639) 2024-11-26T19:15:40,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-26T19:15:40,682 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T19:15:40,682 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33811 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T19:15:40,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33811 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees 2024-11-26T19:15:40,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-26T19:15:40,684 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T19:15:40,685 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T19:15:40,685 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T19:15:40,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-26T19:15:40,837 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-26T19:15:40,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd. 2024-11-26T19:15:40,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd. as already flushing 2024-11-26T19:15:40,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd. 2024-11-26T19:15:40,838 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 082e64a1d02e824a02737c5fd026f5bd, NAME => 'TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:40,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 082e64a1d02e824a02737c5fd026f5bd, NAME => 'TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:40,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 082e64a1d02e824a02737c5fd026f5bd, NAME => 'TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 082e64a1d02e824a02737c5fd026f5bd, NAME => 'TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:40,952 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:40,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58878 deadline: 1732648600941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:40,953 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:40,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58858 deadline: 1732648600941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:40,953 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:40,953 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:40,953 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:40,953 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:40,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58914 deadline: 1732648600942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:40,953 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:40,953 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:40,953 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:40,953 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:40,953 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:40,953 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:40,990 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-26T19:15:40,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd. 2024-11-26T19:15:40,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd. as already flushing 2024-11-26T19:15:40,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd. 2024-11-26T19:15:40,991 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 082e64a1d02e824a02737c5fd026f5bd, NAME => 'TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:40,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 082e64a1d02e824a02737c5fd026f5bd, NAME => 'TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:40,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 082e64a1d02e824a02737c5fd026f5bd, NAME => 'TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 082e64a1d02e824a02737c5fd026f5bd, NAME => 'TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:41,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-26T19:15:41,049 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=284, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp/A/b009e4aade0c410889ba48b66f438038 2024-11-26T19:15:41,056 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp/B/58c6fa3f37ea4cb2a498e2ca35fa8d2f is 50, key is test_row_0/B:col10/1732648538042/Put/seqid=0 2024-11-26T19:15:41,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742238_1414 (size=12301) 2024-11-26T19:15:41,144 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-26T19:15:41,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd. 2024-11-26T19:15:41,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd. as already flushing 2024-11-26T19:15:41,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd. 2024-11-26T19:15:41,144 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 082e64a1d02e824a02737c5fd026f5bd, NAME => 'TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:41,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 082e64a1d02e824a02737c5fd026f5bd, NAME => 'TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:41,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 082e64a1d02e824a02737c5fd026f5bd, NAME => 'TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 082e64a1d02e824a02737c5fd026f5bd, NAME => 'TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:41,297 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-26T19:15:41,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd. 2024-11-26T19:15:41,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd. as already flushing 2024-11-26T19:15:41,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd. 2024-11-26T19:15:41,298 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 082e64a1d02e824a02737c5fd026f5bd, NAME => 'TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:41,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 082e64a1d02e824a02737c5fd026f5bd, NAME => 'TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:41,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 082e64a1d02e824a02737c5fd026f5bd, NAME => 'TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 082e64a1d02e824a02737c5fd026f5bd, NAME => 'TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:41,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-26T19:15:41,451 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-26T19:15:41,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd. 2024-11-26T19:15:41,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd. as already flushing 2024-11-26T19:15:41,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd. 2024-11-26T19:15:41,451 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 082e64a1d02e824a02737c5fd026f5bd, NAME => 'TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:41,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 082e64a1d02e824a02737c5fd026f5bd, NAME => 'TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:41,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 082e64a1d02e824a02737c5fd026f5bd, NAME => 'TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 082e64a1d02e824a02737c5fd026f5bd, NAME => 'TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:41,461 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=284 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp/B/58c6fa3f37ea4cb2a498e2ca35fa8d2f 2024-11-26T19:15:41,469 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp/C/4a34b84bd95e4c088de35b6a07dc2438 is 50, key is test_row_0/C:col10/1732648538042/Put/seqid=0 2024-11-26T19:15:41,470 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:41,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58878 deadline: 1732648601462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:41,470 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:41,470 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:41,471 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:41,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58858 deadline: 1732648601462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:41,471 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:41,471 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:41,471 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:41,471 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:41,471 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:41,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58914 deadline: 1732648601462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:41,471 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:41,471 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:41,471 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:15:41,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742239_1415 (size=12301) 2024-11-26T19:15:41,599 INFO [Thread-1620 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1620. 2024-11-26T19:15:41,599 DEBUG [Thread-1620 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T19:15:41,599 DEBUG [Thread-1620 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:15:41,599 DEBUG [Thread-1620 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:15:41,599 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T19:15:41,601 INFO [Thread-1628 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1628. 2024-11-26T19:15:41,601 DEBUG [Thread-1628 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T19:15:41,601 DEBUG [Thread-1628 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:15:41,601 DEBUG [Thread-1628 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:15:41,601 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T19:15:41,604 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-26T19:15:41,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd. 2024-11-26T19:15:41,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd. as already flushing 2024-11-26T19:15:41,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd. 2024-11-26T19:15:41,604 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 082e64a1d02e824a02737c5fd026f5bd, NAME => 'TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:41,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 082e64a1d02e824a02737c5fd026f5bd, NAME => 'TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:41,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 082e64a1d02e824a02737c5fd026f5bd, NAME => 'TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 082e64a1d02e824a02737c5fd026f5bd, NAME => 'TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:41,607 INFO [Thread-1622 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1622. 2024-11-26T19:15:41,607 DEBUG [Thread-1622 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T19:15:41,608 DEBUG [Thread-1622 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:15:41,608 DEBUG [Thread-1622 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:15:41,608 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T19:15:41,611 INFO [Thread-1624 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1624. 2024-11-26T19:15:41,611 DEBUG [Thread-1624 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T19:15:41,611 DEBUG [Thread-1624 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:15:41,612 DEBUG [Thread-1624 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:15:41,612 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T19:15:41,614 INFO [Thread-1626 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1626. 2024-11-26T19:15:41,614 DEBUG [Thread-1626 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T19:15:41,614 DEBUG [Thread-1626 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:15:41,614 DEBUG [Thread-1626 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:15:41,614 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T19:15:41,757 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-26T19:15:41,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd. 2024-11-26T19:15:41,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd. as already flushing 2024-11-26T19:15:41,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd. 2024-11-26T19:15:41,757 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 082e64a1d02e824a02737c5fd026f5bd, NAME => 'TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:41,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 082e64a1d02e824a02737c5fd026f5bd, NAME => 'TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:41,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 082e64a1d02e824a02737c5fd026f5bd, NAME => 'TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 082e64a1d02e824a02737c5fd026f5bd, NAME => 'TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:41,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-26T19:15:41,874 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=284 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp/C/4a34b84bd95e4c088de35b6a07dc2438 2024-11-26T19:15:41,877 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp/A/b009e4aade0c410889ba48b66f438038 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/A/b009e4aade0c410889ba48b66f438038 2024-11-26T19:15:41,880 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/A/b009e4aade0c410889ba48b66f438038, entries=250, sequenceid=284, filesize=47.5 K 2024-11-26T19:15:41,881 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp/B/58c6fa3f37ea4cb2a498e2ca35fa8d2f as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/B/58c6fa3f37ea4cb2a498e2ca35fa8d2f 2024-11-26T19:15:41,883 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/B/58c6fa3f37ea4cb2a498e2ca35fa8d2f, entries=150, sequenceid=284, filesize=12.0 K 2024-11-26T19:15:41,884 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp/C/4a34b84bd95e4c088de35b6a07dc2438 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/C/4a34b84bd95e4c088de35b6a07dc2438 2024-11-26T19:15:41,886 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/C/4a34b84bd95e4c088de35b6a07dc2438, entries=150, sequenceid=284, filesize=12.0 K 2024-11-26T19:15:41,887 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 082e64a1d02e824a02737c5fd026f5bd in 1660ms, sequenceid=284, compaction requested=true 2024-11-26T19:15:41,887 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 082e64a1d02e824a02737c5fd026f5bd: 2024-11-26T19:15:41,887 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 082e64a1d02e824a02737c5fd026f5bd:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T19:15:41,887 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:15:41,887 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 082e64a1d02e824a02737c5fd026f5bd:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T19:15:41,887 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:15:41,887 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 082e64a1d02e824a02737c5fd026f5bd:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T19:15:41,887 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T19:15:41,887 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T19:15:41,887 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T19:15:41,888 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111516 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T19:15:41,888 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37163 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T19:15:41,888 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1541): 082e64a1d02e824a02737c5fd026f5bd/B is initiating minor compaction (all files) 2024-11-26T19:15:41,888 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1541): 082e64a1d02e824a02737c5fd026f5bd/A is initiating minor compaction (all files) 2024-11-26T19:15:41,888 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 082e64a1d02e824a02737c5fd026f5bd/B in TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd. 2024-11-26T19:15:41,888 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 082e64a1d02e824a02737c5fd026f5bd/A in TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd. 2024-11-26T19:15:41,888 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/B/333d6b85db774a4ebaedc1fdac0fbc6b, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/B/76d1e48a0b9e4f03aa58b0944b0f9bf3, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/B/58c6fa3f37ea4cb2a498e2ca35fa8d2f] into tmpdir=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp, totalSize=36.3 K 2024-11-26T19:15:41,888 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/A/cfb1a3a055d84b64bad2b85692d1b5aa, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/A/a48e5f57dfe74028a060103e0b8ead39, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/A/b009e4aade0c410889ba48b66f438038] into tmpdir=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp, totalSize=108.9 K 2024-11-26T19:15:41,888 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd. 2024-11-26T19:15:41,888 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd. files: [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/A/cfb1a3a055d84b64bad2b85692d1b5aa, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/A/a48e5f57dfe74028a060103e0b8ead39, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/A/b009e4aade0c410889ba48b66f438038] 2024-11-26T19:15:41,889 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting 333d6b85db774a4ebaedc1fdac0fbc6b, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1732648534059 2024-11-26T19:15:41,889 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting cfb1a3a055d84b64bad2b85692d1b5aa, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1732648534059 2024-11-26T19:15:41,889 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting 76d1e48a0b9e4f03aa58b0944b0f9bf3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=269, earliestPutTs=1732648536812 2024-11-26T19:15:41,889 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting a48e5f57dfe74028a060103e0b8ead39, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=269, earliestPutTs=1732648536812 2024-11-26T19:15:41,889 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting 58c6fa3f37ea4cb2a498e2ca35fa8d2f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1732648538029 2024-11-26T19:15:41,889 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting b009e4aade0c410889ba48b66f438038, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1732648538029 2024-11-26T19:15:41,894 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=082e64a1d02e824a02737c5fd026f5bd] 2024-11-26T19:15:41,894 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 082e64a1d02e824a02737c5fd026f5bd#B#compaction#347 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T19:15:41,895 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp/B/e5dc000d88c34feca4c939876391a636 is 50, key is test_row_0/B:col10/1732648538042/Put/seqid=0 2024-11-26T19:15:41,895 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112664451b80e0f7434fb8721211504e9073_082e64a1d02e824a02737c5fd026f5bd store=[table=TestAcidGuarantees family=A region=082e64a1d02e824a02737c5fd026f5bd] 2024-11-26T19:15:41,899 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112664451b80e0f7434fb8721211504e9073_082e64a1d02e824a02737c5fd026f5bd, store=[table=TestAcidGuarantees family=A region=082e64a1d02e824a02737c5fd026f5bd] 2024-11-26T19:15:41,899 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112664451b80e0f7434fb8721211504e9073_082e64a1d02e824a02737c5fd026f5bd because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=082e64a1d02e824a02737c5fd026f5bd] 2024-11-26T19:15:41,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742240_1416 (size=12813) 2024-11-26T19:15:41,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742241_1417 (size=4469) 2024-11-26T19:15:41,910 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-26T19:15:41,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd. 2024-11-26T19:15:41,910 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2902): Flushing 082e64a1d02e824a02737c5fd026f5bd 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-26T19:15:41,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 082e64a1d02e824a02737c5fd026f5bd, store=A 2024-11-26T19:15:41,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:15:41,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 082e64a1d02e824a02737c5fd026f5bd, store=B 2024-11-26T19:15:41,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:15:41,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 082e64a1d02e824a02737c5fd026f5bd, store=C 2024-11-26T19:15:41,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:15:41,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126b5eb33e8d0154e969663863fbc303c28_082e64a1d02e824a02737c5fd026f5bd is 50, key is test_row_0/A:col10/1732648540290/Put/seqid=0 2024-11-26T19:15:41,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742242_1418 (size=12454) 2024-11-26T19:15:42,135 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-26T19:15:42,305 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp/B/e5dc000d88c34feca4c939876391a636 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/B/e5dc000d88c34feca4c939876391a636 2024-11-26T19:15:42,309 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 082e64a1d02e824a02737c5fd026f5bd/B of 082e64a1d02e824a02737c5fd026f5bd into e5dc000d88c34feca4c939876391a636(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T19:15:42,309 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 082e64a1d02e824a02737c5fd026f5bd: 2024-11-26T19:15:42,309 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., storeName=082e64a1d02e824a02737c5fd026f5bd/B, priority=13, startTime=1732648541887; duration=0sec 2024-11-26T19:15:42,309 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T19:15:42,309 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 082e64a1d02e824a02737c5fd026f5bd:B 2024-11-26T19:15:42,310 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T19:15:42,310 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 082e64a1d02e824a02737c5fd026f5bd#A#compaction#348 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T19:15:42,310 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37163 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T19:15:42,310 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1541): 082e64a1d02e824a02737c5fd026f5bd/C is initiating minor compaction (all files) 2024-11-26T19:15:42,310 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 082e64a1d02e824a02737c5fd026f5bd/C in TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd. 2024-11-26T19:15:42,311 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/C/d54e9ff5b6b4458487776e505f9c95e7, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/C/80151d68b0b44e5fafb6fb2a77472eb3, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/C/4a34b84bd95e4c088de35b6a07dc2438] into tmpdir=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp, totalSize=36.3 K 2024-11-26T19:15:42,311 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp/A/c00ce37a1a5244c5a261d6b74bdb5c99 is 175, key is test_row_0/A:col10/1732648538042/Put/seqid=0 2024-11-26T19:15:42,311 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting d54e9ff5b6b4458487776e505f9c95e7, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1732648534059 2024-11-26T19:15:42,311 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting 80151d68b0b44e5fafb6fb2a77472eb3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=269, earliestPutTs=1732648536812 2024-11-26T19:15:42,312 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting 4a34b84bd95e4c088de35b6a07dc2438, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1732648538029 2024-11-26T19:15:42,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742243_1419 (size=31767) 2024-11-26T19:15:42,318 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 082e64a1d02e824a02737c5fd026f5bd#C#compaction#350 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T19:15:42,318 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp/C/d67042ac2e4949fc9e4f9ecf6a72f84f is 50, key is test_row_0/C:col10/1732648538042/Put/seqid=0 2024-11-26T19:15:42,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:42,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742244_1420 (size=12813) 2024-11-26T19:15:42,322 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126b5eb33e8d0154e969663863fbc303c28_082e64a1d02e824a02737c5fd026f5bd to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126b5eb33e8d0154e969663863fbc303c28_082e64a1d02e824a02737c5fd026f5bd 2024-11-26T19:15:42,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp/A/2a5f7de5a6ef459796f166491eecb205, store: [table=TestAcidGuarantees family=A region=082e64a1d02e824a02737c5fd026f5bd] 2024-11-26T19:15:42,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp/A/2a5f7de5a6ef459796f166491eecb205 is 175, key is test_row_0/A:col10/1732648540290/Put/seqid=0 2024-11-26T19:15:42,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742245_1421 (size=31255) 2024-11-26T19:15:42,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(8855): Flush requested on 082e64a1d02e824a02737c5fd026f5bd 2024-11-26T19:15:42,353 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd. as already flushing 2024-11-26T19:15:42,354 INFO [Thread-1611 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1611. 2024-11-26T19:15:42,354 INFO [Thread-1615 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1615. 2024-11-26T19:15:42,354 DEBUG [Thread-1611 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T19:15:42,354 DEBUG [Thread-1615 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T19:15:42,354 DEBUG [Thread-1611 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:15:42,354 DEBUG [Thread-1615 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:15:42,354 DEBUG [Thread-1611 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:15:42,354 DEBUG [Thread-1615 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:15:42,354 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T19:15:42,354 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T19:15:42,484 INFO [Thread-1613 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1613. 2024-11-26T19:15:42,484 DEBUG [Thread-1613 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T19:15:42,484 DEBUG [Thread-1613 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:15:42,484 INFO [Thread-1617 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1617. 2024-11-26T19:15:42,484 INFO [Thread-1609 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1609. 2024-11-26T19:15:42,484 DEBUG [Thread-1613 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:15:42,484 DEBUG [Thread-1617 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T19:15:42,484 DEBUG [Thread-1609 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T19:15:42,484 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T19:15:42,484 DEBUG [Thread-1617 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:15:42,484 DEBUG [Thread-1609 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:15:42,484 DEBUG [Thread-1609 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:15:42,484 DEBUG [Thread-1617 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:15:42,484 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T19:15:42,484 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T19:15:42,718 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp/A/c00ce37a1a5244c5a261d6b74bdb5c99 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/A/c00ce37a1a5244c5a261d6b74bdb5c99 2024-11-26T19:15:42,722 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 082e64a1d02e824a02737c5fd026f5bd/A of 082e64a1d02e824a02737c5fd026f5bd into c00ce37a1a5244c5a261d6b74bdb5c99(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T19:15:42,722 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 082e64a1d02e824a02737c5fd026f5bd: 2024-11-26T19:15:42,722 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., storeName=082e64a1d02e824a02737c5fd026f5bd/A, priority=13, startTime=1732648541887; duration=0sec 2024-11-26T19:15:42,722 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:15:42,722 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 082e64a1d02e824a02737c5fd026f5bd:A 2024-11-26T19:15:42,725 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp/C/d67042ac2e4949fc9e4f9ecf6a72f84f as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/C/d67042ac2e4949fc9e4f9ecf6a72f84f 2024-11-26T19:15:42,727 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=303, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp/A/2a5f7de5a6ef459796f166491eecb205 2024-11-26T19:15:42,728 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 082e64a1d02e824a02737c5fd026f5bd/C of 082e64a1d02e824a02737c5fd026f5bd into d67042ac2e4949fc9e4f9ecf6a72f84f(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T19:15:42,728 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 082e64a1d02e824a02737c5fd026f5bd: 2024-11-26T19:15:42,728 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., storeName=082e64a1d02e824a02737c5fd026f5bd/C, priority=13, startTime=1732648541887; duration=0sec 2024-11-26T19:15:42,728 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:15:42,729 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 082e64a1d02e824a02737c5fd026f5bd:C 2024-11-26T19:15:42,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp/B/16ec3985bd424c63b56c2113350616c6 is 50, key is test_row_0/B:col10/1732648540290/Put/seqid=0 2024-11-26T19:15:42,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742246_1422 (size=12301) 2024-11-26T19:15:42,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-26T19:15:43,136 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=303 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp/B/16ec3985bd424c63b56c2113350616c6 2024-11-26T19:15:43,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp/C/515e8842ce804fa3a2d21e349b69cb7a is 50, key is test_row_0/C:col10/1732648540290/Put/seqid=0 2024-11-26T19:15:43,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742247_1423 (size=12301) 2024-11-26T19:15:43,545 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=303 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp/C/515e8842ce804fa3a2d21e349b69cb7a 2024-11-26T19:15:43,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp/A/2a5f7de5a6ef459796f166491eecb205 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/A/2a5f7de5a6ef459796f166491eecb205 2024-11-26T19:15:43,552 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/A/2a5f7de5a6ef459796f166491eecb205, entries=150, sequenceid=303, filesize=30.5 K 2024-11-26T19:15:43,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp/B/16ec3985bd424c63b56c2113350616c6 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/B/16ec3985bd424c63b56c2113350616c6 2024-11-26T19:15:43,555 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/B/16ec3985bd424c63b56c2113350616c6, entries=150, sequenceid=303, filesize=12.0 K 2024-11-26T19:15:43,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp/C/515e8842ce804fa3a2d21e349b69cb7a as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/C/515e8842ce804fa3a2d21e349b69cb7a 2024-11-26T19:15:43,559 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/C/515e8842ce804fa3a2d21e349b69cb7a, entries=150, sequenceid=303, filesize=12.0 K 2024-11-26T19:15:43,559 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=33.54 KB/34350 for 082e64a1d02e824a02737c5fd026f5bd in 1649ms, sequenceid=303, compaction requested=false 2024-11-26T19:15:43,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2603): Flush status journal for 082e64a1d02e824a02737c5fd026f5bd: 2024-11-26T19:15:43,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd. 2024-11-26T19:15:43,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=106 2024-11-26T19:15:43,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33811 {}] master.HMaster(4169): Remote procedure done, pid=106 2024-11-26T19:15:43,562 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-11-26T19:15:43,562 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.8760 sec 2024-11-26T19:15:43,563 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees in 2.8810 sec 2024-11-26T19:15:44,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-26T19:15:44,862 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T19:15:44,862 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-26T19:15:44,862 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 56 2024-11-26T19:15:44,862 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 47 2024-11-26T19:15:44,862 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 52 2024-11-26T19:15:44,862 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 47 2024-11-26T19:15:44,862 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 54 2024-11-26T19:15:44,862 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-26T19:15:44,862 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-26T19:15:44,862 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 947 2024-11-26T19:15:44,862 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2839 rows 2024-11-26T19:15:44,862 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 950 2024-11-26T19:15:44,862 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2848 rows 2024-11-26T19:15:44,862 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 952 2024-11-26T19:15:44,862 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2856 rows 2024-11-26T19:15:44,862 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 950 2024-11-26T19:15:44,862 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2847 rows 2024-11-26T19:15:44,862 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 946 2024-11-26T19:15:44,862 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2838 rows 2024-11-26T19:15:44,862 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-26T19:15:44,862 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-26T19:15:44,862 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:15:44,862 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:15:44,862 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T19:15:44,863 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33811 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-26T19:15:44,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33811 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-26T19:15:44,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-26T19:15:44,866 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732648544866"}]},"ts":"1732648544866"} 2024-11-26T19:15:44,867 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-26T19:15:44,867 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-26T19:15:44,868 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-26T19:15:44,869 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=109, ppid=108, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=082e64a1d02e824a02737c5fd026f5bd, UNASSIGN}] 2024-11-26T19:15:44,870 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=109, ppid=108, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=082e64a1d02e824a02737c5fd026f5bd, UNASSIGN 2024-11-26T19:15:44,870 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=109 updating hbase:meta row=082e64a1d02e824a02737c5fd026f5bd, regionState=CLOSING, regionLocation=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:44,871 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=109, ppid=108, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=082e64a1d02e824a02737c5fd026f5bd, UNASSIGN because future has completed 2024-11-26T19:15:44,871 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-26T19:15:44,871 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; CloseRegionProcedure 082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970}] 2024-11-26T19:15:44,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-26T19:15:45,023 INFO [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] handler.UnassignRegionHandler(122): Close 082e64a1d02e824a02737c5fd026f5bd 2024-11-26T19:15:45,023 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-26T19:15:45,023 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1722): Closing 082e64a1d02e824a02737c5fd026f5bd, disabling compactions & flushes 2024-11-26T19:15:45,023 INFO [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd. 2024-11-26T19:15:45,024 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd. 2024-11-26T19:15:45,024 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd. after waiting 0 ms 2024-11-26T19:15:45,024 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd. 2024-11-26T19:15:45,024 INFO [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(2902): Flushing 082e64a1d02e824a02737c5fd026f5bd 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-26T19:15:45,024 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 082e64a1d02e824a02737c5fd026f5bd, store=A 2024-11-26T19:15:45,024 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:15:45,024 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 082e64a1d02e824a02737c5fd026f5bd, store=B 2024-11-26T19:15:45,024 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:15:45,024 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 082e64a1d02e824a02737c5fd026f5bd, store=C 2024-11-26T19:15:45,024 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:15:45,029 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112606acc5cc23484fef90bbb9737f44dc0a_082e64a1d02e824a02737c5fd026f5bd is 50, key is test_row_1/A:col10/1732648542482/Put/seqid=0 2024-11-26T19:15:45,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742248_1424 (size=9914) 2024-11-26T19:15:45,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-26T19:15:45,433 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:45,436 INFO [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112606acc5cc23484fef90bbb9737f44dc0a_082e64a1d02e824a02737c5fd026f5bd to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112606acc5cc23484fef90bbb9737f44dc0a_082e64a1d02e824a02737c5fd026f5bd 2024-11-26T19:15:45,437 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp/A/6999a35602de4b1c97c8064b02e371b3, store: [table=TestAcidGuarantees family=A region=082e64a1d02e824a02737c5fd026f5bd] 2024-11-26T19:15:45,437 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp/A/6999a35602de4b1c97c8064b02e371b3 is 175, key is test_row_1/A:col10/1732648542482/Put/seqid=0 2024-11-26T19:15:45,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742249_1425 (size=22561) 2024-11-26T19:15:45,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-26T19:15:45,841 INFO [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=314, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp/A/6999a35602de4b1c97c8064b02e371b3 2024-11-26T19:15:45,846 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp/B/ba462e467ab844e8a1bcadd10f25c9c1 is 50, key is test_row_1/B:col10/1732648542482/Put/seqid=0 2024-11-26T19:15:45,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742250_1426 (size=9857) 2024-11-26T19:15:46,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-26T19:15:46,250 INFO [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=314 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp/B/ba462e467ab844e8a1bcadd10f25c9c1 2024-11-26T19:15:46,255 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp/C/227ae02df8154e8dbebdb8392665374a is 50, key is test_row_1/C:col10/1732648542482/Put/seqid=0 2024-11-26T19:15:46,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742251_1427 (size=9857) 2024-11-26T19:15:46,418 WARN [regionserver/fe76acf732b2:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 6, running: 1 2024-11-26T19:15:46,419 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fe76acf732b2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/A/aa10863e62b74d7a963b18b94263ba99, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/A/f6cd4e46b01d4402b7ea76d3df1439e6, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/A/2bbe6875e5f4473ca1eaeeb78a7e7609, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/A/f3ca20aed6e5408abee89210f3ffa1d9, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/A/cec973f41b5643919ab55d6facce6761, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/A/0e2aabc5ec6b4af88397bbc626bc1b7e, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/A/20a9d62f35e54f7d90060a4137376dbe, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/A/25cd12215aa440659e1a058a9a3b1d98, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/A/448542bb2a654073b3caff8879439129, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/A/e0f8f052966a4c3da4e38cc3d9be7da0, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/A/cd689d7aad9e434e968235e59a87452e, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/A/47f3b7df4b984c63972851f6beeae636, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/A/cfb1a3a055d84b64bad2b85692d1b5aa, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/A/a48e5f57dfe74028a060103e0b8ead39, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/A/b009e4aade0c410889ba48b66f438038] to archive 2024-11-26T19:15:46,420 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fe76acf732b2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-26T19:15:46,421 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fe76acf732b2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/A/aa10863e62b74d7a963b18b94263ba99 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/A/aa10863e62b74d7a963b18b94263ba99 2024-11-26T19:15:46,422 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fe76acf732b2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/A/f6cd4e46b01d4402b7ea76d3df1439e6 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/A/f6cd4e46b01d4402b7ea76d3df1439e6 2024-11-26T19:15:46,423 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fe76acf732b2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/A/2bbe6875e5f4473ca1eaeeb78a7e7609 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/A/2bbe6875e5f4473ca1eaeeb78a7e7609 2024-11-26T19:15:46,424 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fe76acf732b2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/A/f3ca20aed6e5408abee89210f3ffa1d9 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/A/f3ca20aed6e5408abee89210f3ffa1d9 2024-11-26T19:15:46,425 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fe76acf732b2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/A/cec973f41b5643919ab55d6facce6761 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/A/cec973f41b5643919ab55d6facce6761 2024-11-26T19:15:46,425 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fe76acf732b2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/A/0e2aabc5ec6b4af88397bbc626bc1b7e to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/A/0e2aabc5ec6b4af88397bbc626bc1b7e 2024-11-26T19:15:46,426 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fe76acf732b2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/A/20a9d62f35e54f7d90060a4137376dbe to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/A/20a9d62f35e54f7d90060a4137376dbe 2024-11-26T19:15:46,427 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fe76acf732b2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/A/25cd12215aa440659e1a058a9a3b1d98 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/A/25cd12215aa440659e1a058a9a3b1d98 2024-11-26T19:15:46,428 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fe76acf732b2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/A/448542bb2a654073b3caff8879439129 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/A/448542bb2a654073b3caff8879439129 2024-11-26T19:15:46,429 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fe76acf732b2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/A/e0f8f052966a4c3da4e38cc3d9be7da0 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/A/e0f8f052966a4c3da4e38cc3d9be7da0 2024-11-26T19:15:46,429 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fe76acf732b2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/A/cd689d7aad9e434e968235e59a87452e to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/A/cd689d7aad9e434e968235e59a87452e 2024-11-26T19:15:46,430 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fe76acf732b2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/A/47f3b7df4b984c63972851f6beeae636 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/A/47f3b7df4b984c63972851f6beeae636 2024-11-26T19:15:46,431 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fe76acf732b2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/A/cfb1a3a055d84b64bad2b85692d1b5aa to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/A/cfb1a3a055d84b64bad2b85692d1b5aa 2024-11-26T19:15:46,432 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fe76acf732b2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/A/a48e5f57dfe74028a060103e0b8ead39 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/A/a48e5f57dfe74028a060103e0b8ead39 2024-11-26T19:15:46,432 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fe76acf732b2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/A/b009e4aade0c410889ba48b66f438038 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/A/b009e4aade0c410889ba48b66f438038 2024-11-26T19:15:46,435 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fe76acf732b2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/B/6077bbc4704c4f8aaf3450bda07d63ea, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/B/856a61e532ef4b4c99f52e52902bbf15, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/B/9e574ca652374eceaed7ac1547ce6564, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/B/1c6f109693564675b6ade8075652de00, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/B/699b2dcd81384c4c839aba9062adebee, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/B/fa6479c9f4014069b8081db22f2fc084, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/B/31a96a2a4a8d4fd39028234e2c0896f4, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/B/5e6b9b512a3d48ca86c05163bacf8931, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/B/5c1e45e54d5f4766a6ef182c5544fd89, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/B/13460d3ed9b8477d9b5fc92ec379669a, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/B/53c11c5b02fb44b99512e8521712f7e0, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/B/333d6b85db774a4ebaedc1fdac0fbc6b, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/B/9e41b20c2437410fa6d837307c720f05, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/B/76d1e48a0b9e4f03aa58b0944b0f9bf3, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/B/58c6fa3f37ea4cb2a498e2ca35fa8d2f] to archive 2024-11-26T19:15:46,435 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fe76acf732b2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-26T19:15:46,436 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fe76acf732b2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/B/6077bbc4704c4f8aaf3450bda07d63ea to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/B/6077bbc4704c4f8aaf3450bda07d63ea 2024-11-26T19:15:46,437 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fe76acf732b2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/B/856a61e532ef4b4c99f52e52902bbf15 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/B/856a61e532ef4b4c99f52e52902bbf15 2024-11-26T19:15:46,438 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fe76acf732b2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/B/9e574ca652374eceaed7ac1547ce6564 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/B/9e574ca652374eceaed7ac1547ce6564 2024-11-26T19:15:46,439 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fe76acf732b2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/B/1c6f109693564675b6ade8075652de00 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/B/1c6f109693564675b6ade8075652de00 2024-11-26T19:15:46,440 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fe76acf732b2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/B/699b2dcd81384c4c839aba9062adebee to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/B/699b2dcd81384c4c839aba9062adebee 2024-11-26T19:15:46,441 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fe76acf732b2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/B/fa6479c9f4014069b8081db22f2fc084 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/B/fa6479c9f4014069b8081db22f2fc084 2024-11-26T19:15:46,441 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fe76acf732b2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/B/31a96a2a4a8d4fd39028234e2c0896f4 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/B/31a96a2a4a8d4fd39028234e2c0896f4 2024-11-26T19:15:46,442 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fe76acf732b2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/B/5e6b9b512a3d48ca86c05163bacf8931 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/B/5e6b9b512a3d48ca86c05163bacf8931 2024-11-26T19:15:46,443 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fe76acf732b2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/B/5c1e45e54d5f4766a6ef182c5544fd89 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/B/5c1e45e54d5f4766a6ef182c5544fd89 2024-11-26T19:15:46,443 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fe76acf732b2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/B/13460d3ed9b8477d9b5fc92ec379669a to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/B/13460d3ed9b8477d9b5fc92ec379669a 2024-11-26T19:15:46,444 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fe76acf732b2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/B/53c11c5b02fb44b99512e8521712f7e0 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/B/53c11c5b02fb44b99512e8521712f7e0 2024-11-26T19:15:46,445 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fe76acf732b2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/B/333d6b85db774a4ebaedc1fdac0fbc6b to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/B/333d6b85db774a4ebaedc1fdac0fbc6b 2024-11-26T19:15:46,446 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fe76acf732b2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/B/9e41b20c2437410fa6d837307c720f05 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/B/9e41b20c2437410fa6d837307c720f05 2024-11-26T19:15:46,446 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fe76acf732b2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/B/76d1e48a0b9e4f03aa58b0944b0f9bf3 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/B/76d1e48a0b9e4f03aa58b0944b0f9bf3 2024-11-26T19:15:46,447 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fe76acf732b2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/B/58c6fa3f37ea4cb2a498e2ca35fa8d2f to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/B/58c6fa3f37ea4cb2a498e2ca35fa8d2f 2024-11-26T19:15:46,449 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fe76acf732b2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/C/5a09b4d5444b4a6c8039aadea772b401, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/C/a547238000ac4e9b8d8eebedee916dbd, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/C/2d9fbfcec4ef4a38b1a508e06f99281e, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/C/4eeca70933ea407ea9ab96d857e3db89, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/C/4c85c64e413b422197212ce46d79ad66, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/C/30d8ded948704a30bef1e033058b615a, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/C/210e101a7b34413da00b42b491562021, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/C/a252478484474687a535155b03c126b4, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/C/a391bd18f1fd4a65a1c74049bf382942, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/C/016e03b18a9e42ef995014296f618a5b, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/C/9cd31dffa1c84f18a869b2e579c993b4, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/C/d54e9ff5b6b4458487776e505f9c95e7, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/C/27b6b7f7acf949baaed66b10b55ba7d7, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/C/80151d68b0b44e5fafb6fb2a77472eb3, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/C/4a34b84bd95e4c088de35b6a07dc2438] to archive 2024-11-26T19:15:46,450 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fe76acf732b2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-26T19:15:46,451 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fe76acf732b2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/C/5a09b4d5444b4a6c8039aadea772b401 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/C/5a09b4d5444b4a6c8039aadea772b401 2024-11-26T19:15:46,452 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fe76acf732b2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/C/a547238000ac4e9b8d8eebedee916dbd to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/C/a547238000ac4e9b8d8eebedee916dbd 2024-11-26T19:15:46,452 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fe76acf732b2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/C/2d9fbfcec4ef4a38b1a508e06f99281e to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/C/2d9fbfcec4ef4a38b1a508e06f99281e 2024-11-26T19:15:46,453 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fe76acf732b2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/C/4eeca70933ea407ea9ab96d857e3db89 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/C/4eeca70933ea407ea9ab96d857e3db89 2024-11-26T19:15:46,454 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fe76acf732b2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/C/4c85c64e413b422197212ce46d79ad66 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/C/4c85c64e413b422197212ce46d79ad66 2024-11-26T19:15:46,455 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fe76acf732b2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/C/30d8ded948704a30bef1e033058b615a to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/C/30d8ded948704a30bef1e033058b615a 2024-11-26T19:15:46,455 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fe76acf732b2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/C/210e101a7b34413da00b42b491562021 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/C/210e101a7b34413da00b42b491562021 2024-11-26T19:15:46,456 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fe76acf732b2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/C/a252478484474687a535155b03c126b4 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/C/a252478484474687a535155b03c126b4 2024-11-26T19:15:46,457 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fe76acf732b2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/C/a391bd18f1fd4a65a1c74049bf382942 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/C/a391bd18f1fd4a65a1c74049bf382942 2024-11-26T19:15:46,458 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fe76acf732b2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/C/016e03b18a9e42ef995014296f618a5b to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/C/016e03b18a9e42ef995014296f618a5b 2024-11-26T19:15:46,458 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fe76acf732b2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/C/9cd31dffa1c84f18a869b2e579c993b4 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/C/9cd31dffa1c84f18a869b2e579c993b4 2024-11-26T19:15:46,459 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fe76acf732b2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/C/d54e9ff5b6b4458487776e505f9c95e7 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/C/d54e9ff5b6b4458487776e505f9c95e7 2024-11-26T19:15:46,460 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fe76acf732b2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/C/27b6b7f7acf949baaed66b10b55ba7d7 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/C/27b6b7f7acf949baaed66b10b55ba7d7 2024-11-26T19:15:46,460 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fe76acf732b2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/C/80151d68b0b44e5fafb6fb2a77472eb3 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/C/80151d68b0b44e5fafb6fb2a77472eb3 2024-11-26T19:15:46,461 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fe76acf732b2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/C/4a34b84bd95e4c088de35b6a07dc2438 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/C/4a34b84bd95e4c088de35b6a07dc2438 2024-11-26T19:15:46,659 INFO [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=314 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp/C/227ae02df8154e8dbebdb8392665374a 2024-11-26T19:15:46,662 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp/A/6999a35602de4b1c97c8064b02e371b3 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/A/6999a35602de4b1c97c8064b02e371b3 2024-11-26T19:15:46,666 INFO [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/A/6999a35602de4b1c97c8064b02e371b3, entries=100, sequenceid=314, filesize=22.0 K 2024-11-26T19:15:46,666 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp/B/ba462e467ab844e8a1bcadd10f25c9c1 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/B/ba462e467ab844e8a1bcadd10f25c9c1 2024-11-26T19:15:46,669 INFO [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/B/ba462e467ab844e8a1bcadd10f25c9c1, entries=100, sequenceid=314, filesize=9.6 K 2024-11-26T19:15:46,669 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/.tmp/C/227ae02df8154e8dbebdb8392665374a as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/C/227ae02df8154e8dbebdb8392665374a 2024-11-26T19:15:46,673 INFO [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/C/227ae02df8154e8dbebdb8392665374a, entries=100, sequenceid=314, filesize=9.6 K 2024-11-26T19:15:46,674 INFO [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 082e64a1d02e824a02737c5fd026f5bd in 1650ms, sequenceid=314, compaction requested=true 2024-11-26T19:15:46,677 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/recovered.edits/317.seqid, newMaxSeqId=317, maxSeqId=4 2024-11-26T19:15:46,678 INFO [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd. 2024-11-26T19:15:46,678 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1676): Region close journal for 082e64a1d02e824a02737c5fd026f5bd: Waiting for close lock at 1732648545023Running coprocessor pre-close hooks at 1732648545023Disabling compacts and flushes for region at 1732648545023Disabling writes for close at 1732648545024 (+1 ms)Obtaining lock to block concurrent updates at 1732648545024Preparing flush snapshotting stores in 082e64a1d02e824a02737c5fd026f5bd at 1732648545024Finished memstore snapshotting TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1732648545024Flushing stores of TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd. at 1732648545025 (+1 ms)Flushing 082e64a1d02e824a02737c5fd026f5bd/A: creating writer at 1732648545025Flushing mob file 082e64a1d02e824a02737c5fd026f5bd/A: appending metadata at 1732648545029 (+4 ms)Flushing mob file 082e64a1d02e824a02737c5fd026f5bd/A: closing flushed file at 1732648545029Flushing 082e64a1d02e824a02737c5fd026f5bd/A: appending metadata at 1732648545437 (+408 ms)Flushing 082e64a1d02e824a02737c5fd026f5bd/A: closing flushed file at 1732648545437Flushing 082e64a1d02e824a02737c5fd026f5bd/B: creating writer at 1732648545844 (+407 ms)Flushing 082e64a1d02e824a02737c5fd026f5bd/B: appending metadata at 1732648545846 (+2 ms)Flushing 082e64a1d02e824a02737c5fd026f5bd/B: closing flushed file at 1732648545846Flushing 082e64a1d02e824a02737c5fd026f5bd/C: creating writer at 1732648546253 (+407 ms)Flushing 082e64a1d02e824a02737c5fd026f5bd/C: appending metadata at 1732648546255 (+2 ms)Flushing 082e64a1d02e824a02737c5fd026f5bd/C: closing flushed file at 1732648546255Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5d75f0fc: reopening flushed file at 1732648546662 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@59936e9: reopening flushed file at 1732648546666 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5be29607: reopening flushed file at 1732648546669 (+3 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 082e64a1d02e824a02737c5fd026f5bd in 1650ms, sequenceid=314, compaction requested=true at 1732648546674 (+5 ms)Writing region close event to WAL at 1732648546675 (+1 ms)Running coprocessor post-close hooks at 1732648546678 (+3 ms)Closed at 1732648546678 2024-11-26T19:15:46,679 INFO [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] handler.UnassignRegionHandler(157): Closed 082e64a1d02e824a02737c5fd026f5bd 2024-11-26T19:15:46,680 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=109 updating hbase:meta row=082e64a1d02e824a02737c5fd026f5bd, regionState=CLOSED 2024-11-26T19:15:46,681 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=110, ppid=109, state=RUNNABLE, hasLock=false; CloseRegionProcedure 082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 because future has completed 2024-11-26T19:15:46,683 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-11-26T19:15:46,683 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; CloseRegionProcedure 082e64a1d02e824a02737c5fd026f5bd, server=fe76acf732b2,42785,1732648423970 in 1.8110 sec 2024-11-26T19:15:46,684 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=109, resume processing ppid=108 2024-11-26T19:15:46,684 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, ppid=108, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=082e64a1d02e824a02737c5fd026f5bd, UNASSIGN in 1.8140 sec 2024-11-26T19:15:46,686 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-11-26T19:15:46,686 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8170 sec 2024-11-26T19:15:46,687 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732648546686"}]},"ts":"1732648546686"} 2024-11-26T19:15:46,688 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-26T19:15:46,688 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-26T19:15:46,689 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8250 sec 2024-11-26T19:15:47,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-26T19:15:47,012 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-26T19:15:47,013 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33811 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-26T19:15:47,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33811 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T19:15:47,014 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=111, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T19:15:47,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-26T19:15:47,015 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=111, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T19:15:47,017 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd 2024-11-26T19:15:47,019 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/A, FileablePath, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/B, FileablePath, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/C, FileablePath, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/recovered.edits] 2024-11-26T19:15:47,021 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/A/2a5f7de5a6ef459796f166491eecb205 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/A/2a5f7de5a6ef459796f166491eecb205 2024-11-26T19:15:47,022 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/A/6999a35602de4b1c97c8064b02e371b3 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/A/6999a35602de4b1c97c8064b02e371b3 2024-11-26T19:15:47,023 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/A/c00ce37a1a5244c5a261d6b74bdb5c99 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/A/c00ce37a1a5244c5a261d6b74bdb5c99 2024-11-26T19:15:47,024 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/B/16ec3985bd424c63b56c2113350616c6 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/B/16ec3985bd424c63b56c2113350616c6 2024-11-26T19:15:47,025 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/B/ba462e467ab844e8a1bcadd10f25c9c1 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/B/ba462e467ab844e8a1bcadd10f25c9c1 2024-11-26T19:15:47,026 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/B/e5dc000d88c34feca4c939876391a636 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/B/e5dc000d88c34feca4c939876391a636 2024-11-26T19:15:47,028 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/C/227ae02df8154e8dbebdb8392665374a to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/C/227ae02df8154e8dbebdb8392665374a 2024-11-26T19:15:47,028 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/C/515e8842ce804fa3a2d21e349b69cb7a to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/C/515e8842ce804fa3a2d21e349b69cb7a 2024-11-26T19:15:47,029 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/C/d67042ac2e4949fc9e4f9ecf6a72f84f to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/C/d67042ac2e4949fc9e4f9ecf6a72f84f 2024-11-26T19:15:47,031 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/recovered.edits/317.seqid to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd/recovered.edits/317.seqid 2024-11-26T19:15:47,032 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/082e64a1d02e824a02737c5fd026f5bd 2024-11-26T19:15:47,032 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-26T19:15:47,032 DEBUG [PEWorker-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-26T19:15:47,033 DEBUG [PEWorker-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-26T19:15:47,035 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112605d6bdef05c24a409d95833276c93804_082e64a1d02e824a02737c5fd026f5bd to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112605d6bdef05c24a409d95833276c93804_082e64a1d02e824a02737c5fd026f5bd 2024-11-26T19:15:47,036 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112606acc5cc23484fef90bbb9737f44dc0a_082e64a1d02e824a02737c5fd026f5bd to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112606acc5cc23484fef90bbb9737f44dc0a_082e64a1d02e824a02737c5fd026f5bd 2024-11-26T19:15:47,037 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126121c78f26c1846c19663c5b1b1b81af8_082e64a1d02e824a02737c5fd026f5bd to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126121c78f26c1846c19663c5b1b1b81af8_082e64a1d02e824a02737c5fd026f5bd 2024-11-26T19:15:47,038 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126420f5150aa9145689cc39cc41515cb16_082e64a1d02e824a02737c5fd026f5bd to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126420f5150aa9145689cc39cc41515cb16_082e64a1d02e824a02737c5fd026f5bd 2024-11-26T19:15:47,039 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411264b7e60d9caa141c0b25c45807f922f1f_082e64a1d02e824a02737c5fd026f5bd to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411264b7e60d9caa141c0b25c45807f922f1f_082e64a1d02e824a02737c5fd026f5bd 2024-11-26T19:15:47,040 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112655cdd69bebbc4313ae48cf1b1a798a68_082e64a1d02e824a02737c5fd026f5bd to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112655cdd69bebbc4313ae48cf1b1a798a68_082e64a1d02e824a02737c5fd026f5bd 2024-11-26T19:15:47,041 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112660f333b408d545b7b4c3bb84ee87e821_082e64a1d02e824a02737c5fd026f5bd to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112660f333b408d545b7b4c3bb84ee87e821_082e64a1d02e824a02737c5fd026f5bd 2024-11-26T19:15:47,041 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112673279be3234e4c059b326c0360f2f9cd_082e64a1d02e824a02737c5fd026f5bd to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112673279be3234e4c059b326c0360f2f9cd_082e64a1d02e824a02737c5fd026f5bd 2024-11-26T19:15:47,042 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411267daac7e2e2364f11b220af20e7a20eb3_082e64a1d02e824a02737c5fd026f5bd to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411267daac7e2e2364f11b220af20e7a20eb3_082e64a1d02e824a02737c5fd026f5bd 2024-11-26T19:15:47,043 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112699a5b353c63b44adb70b899c9873c715_082e64a1d02e824a02737c5fd026f5bd to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112699a5b353c63b44adb70b899c9873c715_082e64a1d02e824a02737c5fd026f5bd 2024-11-26T19:15:47,044 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126add97a30b4d94d8c910c1e1d099758b9_082e64a1d02e824a02737c5fd026f5bd to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126add97a30b4d94d8c910c1e1d099758b9_082e64a1d02e824a02737c5fd026f5bd 2024-11-26T19:15:47,045 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126b5eb33e8d0154e969663863fbc303c28_082e64a1d02e824a02737c5fd026f5bd to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126b5eb33e8d0154e969663863fbc303c28_082e64a1d02e824a02737c5fd026f5bd 2024-11-26T19:15:47,046 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126f33412f9316b4af487df3679eb452a37_082e64a1d02e824a02737c5fd026f5bd to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126f33412f9316b4af487df3679eb452a37_082e64a1d02e824a02737c5fd026f5bd 2024-11-26T19:15:47,047 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126f5ea86887f8149f6920efdbdfa5db7eb_082e64a1d02e824a02737c5fd026f5bd to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126f5ea86887f8149f6920efdbdfa5db7eb_082e64a1d02e824a02737c5fd026f5bd 2024-11-26T19:15:47,047 DEBUG [PEWorker-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-26T19:15:47,049 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=111, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T19:15:47,051 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-26T19:15:47,053 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-26T19:15:47,054 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=111, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T19:15:47,054 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-26T19:15:47,054 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732648547054"}]},"ts":"9223372036854775807"} 2024-11-26T19:15:47,055 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-26T19:15:47,055 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 082e64a1d02e824a02737c5fd026f5bd, NAME => 'TestAcidGuarantees,,1732648519502.082e64a1d02e824a02737c5fd026f5bd.', STARTKEY => '', ENDKEY => ''}] 2024-11-26T19:15:47,055 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-26T19:15:47,056 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732648547055"}]},"ts":"9223372036854775807"} 2024-11-26T19:15:47,057 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-26T19:15:47,057 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=111, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T19:15:47,058 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 44 msec 2024-11-26T19:15:47,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-26T19:15:47,122 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-26T19:15:47,122 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-26T19:15:47,132 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=252 (was 247) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1331211566_22 at /127.0.0.1:57896 [Waiting for operation #257] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1480762926_22 at /127.0.0.1:57928 [Waiting for operation #236] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1480762926_22 at /127.0.0.1:57926 [Waiting for operation #241] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1331211566_22 at /127.0.0.1:38702 [Waiting for operation #437] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_COMPACTED_FILES_DISCHARGER-regionserver/fe76acf732b2:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=461 (was 449) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=354 (was 390), ProcessCount=11 (was 11), AvailableMemoryMB=5763 (was 5864) 2024-11-26T19:15:47,141 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=252, OpenFileDescriptor=461, MaxFileDescriptor=1048576, SystemLoadAverage=354, ProcessCount=11, AvailableMemoryMB=5763 2024-11-26T19:15:47,142 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33811 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-26T19:15:47,142 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33811 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-26T19:15:47,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33811 {}] procedure2.ProcedureExecutor(1139): Stored pid=112, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-26T19:15:47,144 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=112, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-26T19:15:47,144 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:47,144 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33811 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 112 2024-11-26T19:15:47,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-11-26T19:15:47,145 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=112, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-26T19:15:47,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742252_1428 (size=1061) 2024-11-26T19:15:47,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-11-26T19:15:47,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-11-26T19:15:47,551 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16 2024-11-26T19:15:47,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742253_1429 (size=53) 2024-11-26T19:15:47,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-11-26T19:15:47,956 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T19:15:47,956 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 741bfc4b4e2b320db573e476b103383f, disabling compactions & flushes 2024-11-26T19:15:47,956 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. 2024-11-26T19:15:47,956 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. 2024-11-26T19:15:47,956 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. after waiting 0 ms 2024-11-26T19:15:47,956 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. 2024-11-26T19:15:47,956 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. 2024-11-26T19:15:47,956 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 741bfc4b4e2b320db573e476b103383f: Waiting for close lock at 1732648547956Disabling compacts and flushes for region at 1732648547956Disabling writes for close at 1732648547956Writing region close event to WAL at 1732648547956Closed at 1732648547956 2024-11-26T19:15:47,958 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=112, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-26T19:15:47,958 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732648547958"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732648547958"}]},"ts":"1732648547958"} 2024-11-26T19:15:47,959 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-26T19:15:47,960 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=112, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-26T19:15:47,960 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732648547960"}]},"ts":"1732648547960"} 2024-11-26T19:15:47,961 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-26T19:15:47,961 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=741bfc4b4e2b320db573e476b103383f, ASSIGN}] 2024-11-26T19:15:47,962 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=741bfc4b4e2b320db573e476b103383f, ASSIGN 2024-11-26T19:15:47,963 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=741bfc4b4e2b320db573e476b103383f, ASSIGN; state=OFFLINE, location=fe76acf732b2,42785,1732648423970; forceNewPlan=false, retain=false 2024-11-26T19:15:48,113 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=113 updating hbase:meta row=741bfc4b4e2b320db573e476b103383f, regionState=OPENING, regionLocation=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:48,115 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=741bfc4b4e2b320db573e476b103383f, ASSIGN because future has completed 2024-11-26T19:15:48,115 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE, hasLock=false; OpenRegionProcedure 741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970}] 2024-11-26T19:15:48,269 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. 2024-11-26T19:15:48,269 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(7752): Opening region: {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} 2024-11-26T19:15:48,271 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 741bfc4b4e2b320db573e476b103383f 2024-11-26T19:15:48,271 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T19:15:48,271 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(7794): checking encryption for 741bfc4b4e2b320db573e476b103383f 2024-11-26T19:15:48,271 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(7797): checking classloading for 741bfc4b4e2b320db573e476b103383f 2024-11-26T19:15:48,272 INFO [StoreOpener-741bfc4b4e2b320db573e476b103383f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 741bfc4b4e2b320db573e476b103383f 2024-11-26T19:15:48,273 INFO [StoreOpener-741bfc4b4e2b320db573e476b103383f-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-26T19:15:48,273 INFO [StoreOpener-741bfc4b4e2b320db573e476b103383f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 741bfc4b4e2b320db573e476b103383f columnFamilyName A 2024-11-26T19:15:48,273 DEBUG [StoreOpener-741bfc4b4e2b320db573e476b103383f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:48,274 INFO [StoreOpener-741bfc4b4e2b320db573e476b103383f-1 {}] regionserver.HStore(327): Store=741bfc4b4e2b320db573e476b103383f/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T19:15:48,274 INFO [StoreOpener-741bfc4b4e2b320db573e476b103383f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 741bfc4b4e2b320db573e476b103383f 2024-11-26T19:15:48,275 INFO [StoreOpener-741bfc4b4e2b320db573e476b103383f-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-26T19:15:48,275 INFO [StoreOpener-741bfc4b4e2b320db573e476b103383f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 741bfc4b4e2b320db573e476b103383f columnFamilyName B 2024-11-26T19:15:48,275 DEBUG [StoreOpener-741bfc4b4e2b320db573e476b103383f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:48,275 INFO [StoreOpener-741bfc4b4e2b320db573e476b103383f-1 {}] regionserver.HStore(327): Store=741bfc4b4e2b320db573e476b103383f/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T19:15:48,275 INFO [StoreOpener-741bfc4b4e2b320db573e476b103383f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 741bfc4b4e2b320db573e476b103383f 2024-11-26T19:15:48,276 INFO [StoreOpener-741bfc4b4e2b320db573e476b103383f-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-26T19:15:48,276 INFO [StoreOpener-741bfc4b4e2b320db573e476b103383f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 741bfc4b4e2b320db573e476b103383f columnFamilyName C 2024-11-26T19:15:48,276 DEBUG [StoreOpener-741bfc4b4e2b320db573e476b103383f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:15:48,277 INFO [StoreOpener-741bfc4b4e2b320db573e476b103383f-1 {}] regionserver.HStore(327): Store=741bfc4b4e2b320db573e476b103383f/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T19:15:48,277 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. 2024-11-26T19:15:48,277 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1038): replaying wal for 741bfc4b4e2b320db573e476b103383f 2024-11-26T19:15:48,278 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f 2024-11-26T19:15:48,278 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f 2024-11-26T19:15:48,278 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1048): stopping wal replay for 741bfc4b4e2b320db573e476b103383f 2024-11-26T19:15:48,278 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1060): Cleaning up temporary data for 741bfc4b4e2b320db573e476b103383f 2024-11-26T19:15:48,279 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-26T19:15:48,280 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1093): writing seq id for 741bfc4b4e2b320db573e476b103383f 2024-11-26T19:15:48,281 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T19:15:48,281 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1114): Opened 741bfc4b4e2b320db573e476b103383f; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71581186, jitterRate=0.06664279103279114}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-26T19:15:48,281 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 741bfc4b4e2b320db573e476b103383f 2024-11-26T19:15:48,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-11-26T19:15:48,282 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1006): Region open journal for 741bfc4b4e2b320db573e476b103383f: Running coprocessor pre-open hook at 1732648548271Writing region info on filesystem at 1732648548271Initializing all the Stores at 1732648548272 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732648548272Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732648548272Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732648548272Cleaning up temporary data from old regions at 1732648548278 (+6 ms)Running coprocessor post-open hooks at 1732648548281 (+3 ms)Region opened successfully at 1732648548282 (+1 ms) 2024-11-26T19:15:48,283 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., pid=114, masterSystemTime=1732648548267 2024-11-26T19:15:48,284 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. 2024-11-26T19:15:48,284 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. 2024-11-26T19:15:48,285 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=113 updating hbase:meta row=741bfc4b4e2b320db573e476b103383f, regionState=OPEN, openSeqNum=2, regionLocation=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:48,286 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=114, ppid=113, state=RUNNABLE, hasLock=false; OpenRegionProcedure 741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 because future has completed 2024-11-26T19:15:48,287 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-11-26T19:15:48,288 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; OpenRegionProcedure 741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 in 171 msec 2024-11-26T19:15:48,289 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=113, resume processing ppid=112 2024-11-26T19:15:48,289 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, ppid=112, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=741bfc4b4e2b320db573e476b103383f, ASSIGN in 327 msec 2024-11-26T19:15:48,290 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=112, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-26T19:15:48,290 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732648548290"}]},"ts":"1732648548290"} 2024-11-26T19:15:48,291 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-26T19:15:48,291 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=112, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-26T19:15:48,292 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1490 sec 2024-11-26T19:15:49,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-11-26T19:15:49,302 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-26T19:15:49,303 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16af2234, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:15:49,303 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fe76acf732b2,33811,-1 for getting cluster id 2024-11-26T19:15:49,303 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T19:15:49,304 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3cec4e24-71db-4cd0-b63a-4ea30a9b160b' 2024-11-26T19:15:49,304 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T19:15:49,304 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3cec4e24-71db-4cd0-b63a-4ea30a9b160b" 2024-11-26T19:15:49,304 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@548c9cfb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:15:49,304 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fe76acf732b2,33811,-1] 2024-11-26T19:15:49,305 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T19:15:49,305 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:15:49,306 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45378, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T19:15:49,306 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1aad8fb2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:15:49,306 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T19:15:49,307 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fe76acf732b2,42785,1732648423970, seqNum=-1] 2024-11-26T19:15:49,307 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T19:15:49,308 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36684, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T19:15:49,309 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-26T19:15:49,309 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is fe76acf732b2,33811,1732648423099 2024-11-26T19:15:49,309 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@3ccfab47 2024-11-26T19:15:49,309 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-26T19:15:49,310 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45388, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-26T19:15:49,311 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@695dbdde, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:15:49,311 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fe76acf732b2,33811,-1 for getting cluster id 2024-11-26T19:15:49,311 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T19:15:49,312 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3cec4e24-71db-4cd0-b63a-4ea30a9b160b' 2024-11-26T19:15:49,312 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T19:15:49,312 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3cec4e24-71db-4cd0-b63a-4ea30a9b160b" 2024-11-26T19:15:49,312 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f54cc29, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:15:49,312 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fe76acf732b2,33811,-1] 2024-11-26T19:15:49,312 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T19:15:49,312 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:15:49,313 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45406, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T19:15:49,313 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6261460e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:15:49,314 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@594674bb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:15:49,314 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fe76acf732b2,33811,-1 for getting cluster id 2024-11-26T19:15:49,314 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T19:15:49,314 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3cec4e24-71db-4cd0-b63a-4ea30a9b160b' 2024-11-26T19:15:49,315 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T19:15:49,315 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3cec4e24-71db-4cd0-b63a-4ea30a9b160b" 2024-11-26T19:15:49,315 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b650b63, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:15:49,315 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fe76acf732b2,33811,-1] 2024-11-26T19:15:49,315 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T19:15:49,315 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:15:49,316 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45418, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T19:15:49,316 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13b23b7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:15:49,316 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@119ad987, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:15:49,317 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fe76acf732b2,33811,-1 for getting cluster id 2024-11-26T19:15:49,317 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T19:15:49,317 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3cec4e24-71db-4cd0-b63a-4ea30a9b160b' 2024-11-26T19:15:49,317 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T19:15:49,317 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3cec4e24-71db-4cd0-b63a-4ea30a9b160b" 2024-11-26T19:15:49,317 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7476bbf2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:15:49,317 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fe76acf732b2,33811,-1] 2024-11-26T19:15:49,317 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T19:15:49,318 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:15:49,318 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45430, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T19:15:49,318 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15060d78, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:15:49,319 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26c4ef34, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:15:49,319 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fe76acf732b2,33811,-1 for getting cluster id 2024-11-26T19:15:49,319 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T19:15:49,319 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3cec4e24-71db-4cd0-b63a-4ea30a9b160b' 2024-11-26T19:15:49,319 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T19:15:49,319 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3cec4e24-71db-4cd0-b63a-4ea30a9b160b" 2024-11-26T19:15:49,320 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4284573d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:15:49,320 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fe76acf732b2,33811,-1] 2024-11-26T19:15:49,320 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T19:15:49,320 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:15:49,320 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45452, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T19:15:49,321 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@427af512, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:15:49,321 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2fd1f545, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:15:49,321 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fe76acf732b2,33811,-1 for getting cluster id 2024-11-26T19:15:49,321 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T19:15:49,322 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3cec4e24-71db-4cd0-b63a-4ea30a9b160b' 2024-11-26T19:15:49,322 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T19:15:49,322 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3cec4e24-71db-4cd0-b63a-4ea30a9b160b" 2024-11-26T19:15:49,322 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16205f57, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:15:49,322 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fe76acf732b2,33811,-1] 2024-11-26T19:15:49,322 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T19:15:49,322 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:15:49,323 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45466, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T19:15:49,323 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@bab2864, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:15:49,324 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27ae0787, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:15:49,324 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fe76acf732b2,33811,-1 for getting cluster id 2024-11-26T19:15:49,324 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T19:15:49,324 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3cec4e24-71db-4cd0-b63a-4ea30a9b160b' 2024-11-26T19:15:49,324 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T19:15:49,324 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3cec4e24-71db-4cd0-b63a-4ea30a9b160b" 2024-11-26T19:15:49,325 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e81cb0b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:15:49,325 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fe76acf732b2,33811,-1] 2024-11-26T19:15:49,325 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T19:15:49,325 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:15:49,325 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45484, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T19:15:49,326 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11a59858, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:15:49,326 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1627c26b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:15:49,326 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fe76acf732b2,33811,-1 for getting cluster id 2024-11-26T19:15:49,326 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T19:15:49,327 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3cec4e24-71db-4cd0-b63a-4ea30a9b160b' 2024-11-26T19:15:49,327 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T19:15:49,327 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3cec4e24-71db-4cd0-b63a-4ea30a9b160b" 2024-11-26T19:15:49,327 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6449638c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:15:49,327 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fe76acf732b2,33811,-1] 2024-11-26T19:15:49,327 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T19:15:49,328 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:15:49,328 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45500, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T19:15:49,329 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55e6171, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:15:49,329 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5993889b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:15:49,329 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fe76acf732b2,33811,-1 for getting cluster id 2024-11-26T19:15:49,329 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T19:15:49,330 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3cec4e24-71db-4cd0-b63a-4ea30a9b160b' 2024-11-26T19:15:49,330 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T19:15:49,330 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3cec4e24-71db-4cd0-b63a-4ea30a9b160b" 2024-11-26T19:15:49,330 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78dbc9c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:15:49,330 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fe76acf732b2,33811,-1] 2024-11-26T19:15:49,330 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T19:15:49,330 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:15:49,331 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45518, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T19:15:49,331 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29bc7da5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:15:49,331 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@117399b0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:15:49,331 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fe76acf732b2,33811,-1 for getting cluster id 2024-11-26T19:15:49,332 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T19:15:49,332 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3cec4e24-71db-4cd0-b63a-4ea30a9b160b' 2024-11-26T19:15:49,332 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T19:15:49,332 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3cec4e24-71db-4cd0-b63a-4ea30a9b160b" 2024-11-26T19:15:49,332 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21787349, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:15:49,332 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fe76acf732b2,33811,-1] 2024-11-26T19:15:49,332 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T19:15:49,333 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:15:49,333 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45528, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T19:15:49,333 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12a3328a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:15:49,334 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b33a176, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:15:49,334 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fe76acf732b2,33811,-1 for getting cluster id 2024-11-26T19:15:49,334 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T19:15:49,335 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3cec4e24-71db-4cd0-b63a-4ea30a9b160b' 2024-11-26T19:15:49,335 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T19:15:49,335 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3cec4e24-71db-4cd0-b63a-4ea30a9b160b" 2024-11-26T19:15:49,335 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b01587c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:15:49,335 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fe76acf732b2,33811,-1] 2024-11-26T19:15:49,335 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T19:15:49,335 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:15:49,336 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45550, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T19:15:49,336 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29fdca21, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:15:49,336 DEBUG [Thread-1942 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T19:15:49,336 DEBUG [Thread-1945 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T19:15:49,337 DEBUG [Thread-1947 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T19:15:49,337 DEBUG [Thread-1936 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T19:15:49,337 DEBUG [Thread-1949 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T19:15:49,337 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fe76acf732b2,42785,1732648423970, seqNum=-1] 2024-11-26T19:15:49,337 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fe76acf732b2,42785,1732648423970, seqNum=-1] 2024-11-26T19:15:49,337 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fe76acf732b2,42785,1732648423970, seqNum=-1] 2024-11-26T19:15:49,337 DEBUG [Thread-1938 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T19:15:49,337 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T19:15:49,337 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T19:15:49,338 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T19:15:49,338 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fe76acf732b2,42785,1732648423970, seqNum=-1] 2024-11-26T19:15:49,338 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fe76acf732b2,42785,1732648423970, seqNum=-1] 2024-11-26T19:15:49,338 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T19:15:49,338 DEBUG [Thread-1934 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T19:15:49,338 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T19:15:49,338 DEBUG [Thread-1940 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T19:15:49,338 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36700, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T19:15:49,338 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fe76acf732b2,42785,1732648423970, seqNum=-1] 2024-11-26T19:15:49,339 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T19:15:49,339 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fe76acf732b2,42785,1732648423970, seqNum=-1] 2024-11-26T19:15:49,339 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T19:15:49,339 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36730, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T19:15:49,339 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36712, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T19:15:49,339 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fe76acf732b2,42785,1732648423970, seqNum=-1] 2024-11-26T19:15:49,339 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36710, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T19:15:49,339 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T19:15:49,340 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36744, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T19:15:49,340 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36770, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T19:15:49,340 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36726, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T19:15:49,341 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T19:15:49,341 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2] 2024-11-26T19:15:49,341 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36758, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T19:15:49,341 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2] 2024-11-26T19:15:49,341 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2] 2024-11-26T19:15:49,342 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2] 2024-11-26T19:15:49,342 DEBUG [Thread-1951 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T19:15:49,342 DEBUG [Thread-1953 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T19:15:49,342 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2] 2024-11-26T19:15:49,343 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2] 2024-11-26T19:15:49,343 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fe76acf732b2,42785,1732648423970, seqNum=-1] 2024-11-26T19:15:49,343 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2] 2024-11-26T19:15:49,343 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T19:15:49,344 DEBUG [Thread-1951 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fe76acf732b2,42785,1732648423970, seqNum=-1] 2024-11-26T19:15:49,344 DEBUG [Thread-1951 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T19:15:49,344 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36782, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T19:15:49,344 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2] 2024-11-26T19:15:49,344 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36796, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T19:15:49,345 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2] 2024-11-26T19:15:49,346 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2] 2024-11-26T19:15:49,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] procedure2.ProcedureExecutor(1139): Stored pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees 2024-11-26T19:15:49,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-26T19:15:49,348 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T19:15:49,349 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T19:15:49,349 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T19:15:49,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(8855): Flush requested on 741bfc4b4e2b320db573e476b103383f 2024-11-26T19:15:49,352 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 741bfc4b4e2b320db573e476b103383f 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-26T19:15:49,353 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 741bfc4b4e2b320db573e476b103383f, store=A 2024-11-26T19:15:49,353 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:15:49,353 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 741bfc4b4e2b320db573e476b103383f, store=B 2024-11-26T19:15:49,353 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:15:49,353 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 741bfc4b4e2b320db573e476b103383f, store=C 2024-11-26T19:15:49,353 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:15:49,373 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/A/cc13134c158940ce9742b4f24755e256 is 50, key is test_row_1/A:col10/1732648549351/Put/seqid=0 2024-11-26T19:15:49,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742254_1430 (size=16677) 2024-11-26T19:15:49,398 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=409, cellSize=18725, totalHeapSize=42776, min timestamp=1732648549366, max timestamp=1732648549394Num uniques 859; 2024-11-26T19:15:49,399 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=18725, suffix heap size=117064,new segment heap size=42776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=409 2024-11-26T19:15:49,399 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=409, cellSize=18725, totalHeapSize=42776, min timestamp=1732648549366, max timestamp=1732648549394Num uniques 859; 2024-11-26T19:15:49,399 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=18725, suffix heap size=117064,new segment heap size=42776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=409 2024-11-26T19:15:49,401 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=409, cellSize=18725, totalHeapSize=42776, min timestamp=1732648549366, max timestamp=1732648549394Num uniques 859; 2024-11-26T19:15:49,401 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=18725, suffix heap size=117064,new segment heap size=42776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=409 2024-11-26T19:15:49,432 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:49,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36744 deadline: 1732648609427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:49,433 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:49,433 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:49,433 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:49,442 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:49,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36712 deadline: 1732648609429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:49,443 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:49,443 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:49,443 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:49,443 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:49,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36770 deadline: 1732648609431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:49,443 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:49,443 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:49,443 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:49,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36758 deadline: 1732648609432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:49,443 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:49,443 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:49,444 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:49,444 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:49,444 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:49,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36730 deadline: 1732648609433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:49,446 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:49,446 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:49,446 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:49,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-26T19:15:49,501 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-26T19:15:49,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. 2024-11-26T19:15:49,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. as already flushing 2024-11-26T19:15:49,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. 2024-11-26T19:15:49,502 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:49,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:49,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:49,551 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:49,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36744 deadline: 1732648609541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:49,551 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:49,551 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:49,551 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:49,561 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:49,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36712 deadline: 1732648609551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:49,562 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:49,562 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:49,562 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:49,562 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:49,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36770 deadline: 1732648609552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:49,562 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:49,562 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:49,562 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:49,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36758 deadline: 1732648609552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:49,562 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:49,562 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:49,562 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:49,562 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:49,562 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:49,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36730 deadline: 1732648609552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:49,563 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:49,563 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:49,563 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:49,654 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-26T19:15:49,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. 2024-11-26T19:15:49,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. as already flushing 2024-11-26T19:15:49,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. 2024-11-26T19:15:49,655 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:49,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:49,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:49,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-26T19:15:49,769 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:49,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36744 deadline: 1732648609762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:49,769 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:49,769 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:49,769 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:49,782 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:49,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36712 deadline: 1732648609772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:49,783 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:49,783 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:49,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36770 deadline: 1732648609772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:49,783 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:49,783 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:49,783 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:49,783 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:49,783 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:49,783 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:49,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36758 deadline: 1732648609772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:49,784 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:49,784 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:49,784 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:49,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36730 deadline: 1732648609772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:49,784 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:49,784 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:49,784 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:49,784 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:49,791 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/A/cc13134c158940ce9742b4f24755e256 2024-11-26T19:15:49,807 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-26T19:15:49,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. 2024-11-26T19:15:49,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. as already flushing 2024-11-26T19:15:49,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. 2024-11-26T19:15:49,808 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:49,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:49,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:49,810 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/B/12c9fed4ceb34602bb4b2c78e79553e8 is 50, key is test_row_1/B:col10/1732648549351/Put/seqid=0 2024-11-26T19:15:49,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742255_1431 (size=9657) 2024-11-26T19:15:49,960 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-26T19:15:49,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. 2024-11-26T19:15:49,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. as already flushing 2024-11-26T19:15:49,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. 2024-11-26T19:15:49,961 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:49,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:49,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:49,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-26T19:15:50,079 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:50,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36744 deadline: 1732648610072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:50,079 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:50,079 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:50,079 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:50,101 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:50,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36712 deadline: 1732648610092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:50,101 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:50,101 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:50,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36770 deadline: 1732648610092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:50,101 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:50,101 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:50,101 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:50,101 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:50,102 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:50,102 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:50,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36758 deadline: 1732648610092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:50,102 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:50,102 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:50,102 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:50,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36730 deadline: 1732648610092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:50,102 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:50,102 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:50,102 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:50,102 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:50,113 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-26T19:15:50,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. 2024-11-26T19:15:50,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. as already flushing 2024-11-26T19:15:50,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. 2024-11-26T19:15:50,113 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:50,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:50,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:50,214 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/B/12c9fed4ceb34602bb4b2c78e79553e8 2024-11-26T19:15:50,237 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/C/de6ca40c5e004b61a0dce3e14c1b8797 is 50, key is test_row_1/C:col10/1732648549351/Put/seqid=0 2024-11-26T19:15:50,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742256_1432 (size=9657) 2024-11-26T19:15:50,266 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-26T19:15:50,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. 2024-11-26T19:15:50,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. as already flushing 2024-11-26T19:15:50,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. 2024-11-26T19:15:50,266 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:50,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:50,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:50,419 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-26T19:15:50,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. 2024-11-26T19:15:50,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. as already flushing 2024-11-26T19:15:50,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. 2024-11-26T19:15:50,419 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:50,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:50,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:50,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-26T19:15:50,572 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-26T19:15:50,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. 2024-11-26T19:15:50,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. as already flushing 2024-11-26T19:15:50,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. 2024-11-26T19:15:50,573 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:50,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:50,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:50,590 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:50,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36744 deadline: 1732648610582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:50,590 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:50,590 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:50,590 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:50,618 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:50,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36712 deadline: 1732648610611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:50,618 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:50,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36770 deadline: 1732648610612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:50,619 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:50,619 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:50,619 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:50,619 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:50,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36730 deadline: 1732648610612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:50,619 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:50,619 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:50,619 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:50,619 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:50,619 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:50,619 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:50,619 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:50,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36758 deadline: 1732648610612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:50,633 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:50,633 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:50,633 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:50,641 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/C/de6ca40c5e004b61a0dce3e14c1b8797 2024-11-26T19:15:50,651 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/A/cc13134c158940ce9742b4f24755e256 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/A/cc13134c158940ce9742b4f24755e256 2024-11-26T19:15:50,655 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/A/cc13134c158940ce9742b4f24755e256, entries=250, sequenceid=11, filesize=16.3 K 2024-11-26T19:15:50,656 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/B/12c9fed4ceb34602bb4b2c78e79553e8 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/B/12c9fed4ceb34602bb4b2c78e79553e8 2024-11-26T19:15:50,660 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/B/12c9fed4ceb34602bb4b2c78e79553e8, entries=100, sequenceid=11, filesize=9.4 K 2024-11-26T19:15:50,661 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/C/de6ca40c5e004b61a0dce3e14c1b8797 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/C/de6ca40c5e004b61a0dce3e14c1b8797 2024-11-26T19:15:50,664 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/C/de6ca40c5e004b61a0dce3e14c1b8797, entries=100, sequenceid=11, filesize=9.4 K 2024-11-26T19:15:50,665 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for 741bfc4b4e2b320db573e476b103383f in 1313ms, sequenceid=11, compaction requested=false 2024-11-26T19:15:50,665 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 741bfc4b4e2b320db573e476b103383f: 2024-11-26T19:15:50,725 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-26T19:15:50,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. 2024-11-26T19:15:50,725 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2902): Flushing 741bfc4b4e2b320db573e476b103383f 3/3 column families, dataSize=140.89 KB heapSize=381.47 KB 2024-11-26T19:15:50,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 741bfc4b4e2b320db573e476b103383f, store=A 2024-11-26T19:15:50,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T19:15:50,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 741bfc4b4e2b320db573e476b103383f, store=B 2024-11-26T19:15:50,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T19:15:50,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 741bfc4b4e2b320db573e476b103383f, store=C 2024-11-26T19:15:50,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T19:15:50,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/A/f55c1a2a88a14a7dbd27c6dc4a6f6498 is 50, key is test_row_0/A:col10/1732648549426/Put/seqid=0 2024-11-26T19:15:50,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742257_1433 (size=12001) 2024-11-26T19:15:51,139 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=44 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/A/f55c1a2a88a14a7dbd27c6dc4a6f6498 2024-11-26T19:15:51,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/B/b41d94fff91f46b6997e09703e8ee195 is 50, key is test_row_0/B:col10/1732648549426/Put/seqid=0 2024-11-26T19:15:51,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742258_1434 (size=12001) 2024-11-26T19:15:51,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-26T19:15:51,551 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=44 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/B/b41d94fff91f46b6997e09703e8ee195 2024-11-26T19:15:51,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/C/feb26fd9308a4719b36e217fc208430c is 50, key is test_row_0/C:col10/1732648549426/Put/seqid=0 2024-11-26T19:15:51,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742259_1435 (size=12001) 2024-11-26T19:15:51,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(8855): Flush requested on 741bfc4b4e2b320db573e476b103383f 2024-11-26T19:15:51,611 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. as already flushing 2024-11-26T19:15:51,643 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:51,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36744 deadline: 1732648611633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:51,644 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:51,644 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:51,644 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:51,648 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:51,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36712 deadline: 1732648611641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:51,649 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:51,649 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:51,649 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:51,659 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:51,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36758 deadline: 1732648611642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:51,660 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:51,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36770 deadline: 1732648611642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:51,660 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:51,660 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:51,660 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:51,660 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:51,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36730 deadline: 1732648611643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:51,660 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to fe76acf732b2,42785,1732648423970 for 'test_row_2' in 741bfc4b4e2b320db573e476b103383f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2228 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T19:15:51,661 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:51,661 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:51,661 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:51,661 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:51,661 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:51,661 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:51,764 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:51,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36744 deadline: 1732648611752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:51,765 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:51,765 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:51,765 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:51,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36712 deadline: 1732648611752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:51,765 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:51,765 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:51,765 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:51,765 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:51,783 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:51,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36770 deadline: 1732648611771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:51,783 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:51,783 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:51,783 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:51,783 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:51,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36730 deadline: 1732648611771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:51,783 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:51,783 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:51,783 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:51,962 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=44 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/C/feb26fd9308a4719b36e217fc208430c 2024-11-26T19:15:51,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/A/f55c1a2a88a14a7dbd27c6dc4a6f6498 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/A/f55c1a2a88a14a7dbd27c6dc4a6f6498 2024-11-26T19:15:51,970 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/A/f55c1a2a88a14a7dbd27c6dc4a6f6498, entries=150, sequenceid=44, filesize=11.7 K 2024-11-26T19:15:51,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/B/b41d94fff91f46b6997e09703e8ee195 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/B/b41d94fff91f46b6997e09703e8ee195 2024-11-26T19:15:51,974 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/B/b41d94fff91f46b6997e09703e8ee195, entries=150, sequenceid=44, filesize=11.7 K 2024-11-26T19:15:51,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/C/feb26fd9308a4719b36e217fc208430c as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/C/feb26fd9308a4719b36e217fc208430c 2024-11-26T19:15:51,977 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:51,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36744 deadline: 1732648611972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:51,978 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:51,978 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:51,978 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:51,978 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:51,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36712 deadline: 1732648611972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:51,978 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:51,978 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:51,978 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:51,978 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/C/feb26fd9308a4719b36e217fc208430c, entries=150, sequenceid=44, filesize=11.7 K 2024-11-26T19:15:51,979 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~381.42 KB/390576, currentSize=46.96 KB/48090 for 741bfc4b4e2b320db573e476b103383f in 1254ms, sequenceid=44, compaction requested=false 2024-11-26T19:15:51,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2603): Flush status journal for 741bfc4b4e2b320db573e476b103383f: 2024-11-26T19:15:51,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. 2024-11-26T19:15:51,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=116 2024-11-26T19:15:51,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster(4169): Remote procedure done, pid=116 2024-11-26T19:15:51,982 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-11-26T19:15:51,982 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6310 sec 2024-11-26T19:15:51,984 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees in 2.6410 sec 2024-11-26T19:15:51,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(8855): Flush requested on 741bfc4b4e2b320db573e476b103383f 2024-11-26T19:15:51,993 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 741bfc4b4e2b320db573e476b103383f 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-26T19:15:51,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 741bfc4b4e2b320db573e476b103383f, store=A 2024-11-26T19:15:51,994 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:15:51,994 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 741bfc4b4e2b320db573e476b103383f, store=B 2024-11-26T19:15:51,994 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:15:51,994 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 741bfc4b4e2b320db573e476b103383f, store=C 2024-11-26T19:15:51,994 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:15:51,998 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/A/23a12e07036e4c2985e6151fc47168b6 is 50, key is test_row_0/A:col10/1732648551639/Put/seqid=0 2024-11-26T19:15:52,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742260_1436 (size=14341) 2024-11-26T19:15:52,007 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=55 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/A/23a12e07036e4c2985e6151fc47168b6 2024-11-26T19:15:52,013 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/B/d86ff56850bb415c845fe379281a2266 is 50, key is test_row_0/B:col10/1732648551639/Put/seqid=0 2024-11-26T19:15:52,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742261_1437 (size=12001) 2024-11-26T19:15:52,095 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732648552056, max timestamp=1732648552091Num uniques 859; 2024-11-26T19:15:52,095 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-26T19:15:52,095 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732648552056, max timestamp=1732648552091Num uniques 859; 2024-11-26T19:15:52,096 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-26T19:15:52,096 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732648552056, max timestamp=1732648552091Num uniques 859; 2024-11-26T19:15:52,096 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-26T19:15:52,177 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:52,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36730 deadline: 1732648612169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:52,177 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:52,177 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:52,177 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:52,186 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:52,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36770 deadline: 1732648612170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:52,187 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:52,187 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:52,187 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:52,199 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-26T19:15:52,295 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:52,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36744 deadline: 1732648612282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:52,295 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:52,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36712 deadline: 1732648612282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:52,295 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:52,295 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:52,296 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:52,296 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:52,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36730 deadline: 1732648612282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:52,296 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:52,296 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:52,296 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:52,296 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:52,296 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:52,296 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:52,302 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:52,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36770 deadline: 1732648612292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:52,302 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:52,302 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:52,302 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:52,417 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=55 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/B/d86ff56850bb415c845fe379281a2266 2024-11-26T19:15:52,424 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/C/d179307cb34c4305a24c407e63432847 is 50, key is test_row_0/C:col10/1732648551639/Put/seqid=0 2024-11-26T19:15:52,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742262_1438 (size=12001) 2024-11-26T19:15:52,512 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:52,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36730 deadline: 1732648612502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:52,512 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:52,512 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:52,512 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:52,524 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:52,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36770 deadline: 1732648612512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:52,525 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:52,525 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:52,525 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:52,813 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:52,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36744 deadline: 1732648612802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:52,814 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:52,814 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:52,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36712 deadline: 1732648612802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:52,814 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:52,814 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:52,814 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:52,814 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:52,814 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:52,828 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=55 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/C/d179307cb34c4305a24c407e63432847 2024-11-26T19:15:52,833 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/A/23a12e07036e4c2985e6151fc47168b6 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/A/23a12e07036e4c2985e6151fc47168b6 2024-11-26T19:15:52,835 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:52,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36730 deadline: 1732648612821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:52,836 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:52,836 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:52,836 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:52,836 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:52,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36770 deadline: 1732648612832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:52,836 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:52,836 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:52,836 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:52,838 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/A/23a12e07036e4c2985e6151fc47168b6, entries=200, sequenceid=55, filesize=14.0 K 2024-11-26T19:15:52,838 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/B/d86ff56850bb415c845fe379281a2266 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/B/d86ff56850bb415c845fe379281a2266 2024-11-26T19:15:52,842 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/B/d86ff56850bb415c845fe379281a2266, entries=150, sequenceid=55, filesize=11.7 K 2024-11-26T19:15:52,842 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/C/d179307cb34c4305a24c407e63432847 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/C/d179307cb34c4305a24c407e63432847 2024-11-26T19:15:52,845 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/C/d179307cb34c4305a24c407e63432847, entries=150, sequenceid=55, filesize=11.7 K 2024-11-26T19:15:52,846 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 741bfc4b4e2b320db573e476b103383f in 853ms, sequenceid=55, compaction requested=true 2024-11-26T19:15:52,846 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 741bfc4b4e2b320db573e476b103383f: 2024-11-26T19:15:52,846 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 741bfc4b4e2b320db573e476b103383f:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T19:15:52,846 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:15:52,846 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 741bfc4b4e2b320db573e476b103383f:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T19:15:52,846 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:15:52,846 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T19:15:52,846 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 741bfc4b4e2b320db573e476b103383f:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T19:15:52,846 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T19:15:52,846 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T19:15:52,847 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T19:15:52,847 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 43019 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T19:15:52,847 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1541): 741bfc4b4e2b320db573e476b103383f/B is initiating minor compaction (all files) 2024-11-26T19:15:52,847 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 741bfc4b4e2b320db573e476b103383f/B in TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. 2024-11-26T19:15:52,847 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1541): 741bfc4b4e2b320db573e476b103383f/A is initiating minor compaction (all files) 2024-11-26T19:15:52,847 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/B/12c9fed4ceb34602bb4b2c78e79553e8, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/B/b41d94fff91f46b6997e09703e8ee195, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/B/d86ff56850bb415c845fe379281a2266] into tmpdir=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp, totalSize=32.9 K 2024-11-26T19:15:52,847 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 741bfc4b4e2b320db573e476b103383f/A in TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. 2024-11-26T19:15:52,848 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/A/cc13134c158940ce9742b4f24755e256, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/A/f55c1a2a88a14a7dbd27c6dc4a6f6498, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/A/23a12e07036e4c2985e6151fc47168b6] into tmpdir=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp, totalSize=42.0 K 2024-11-26T19:15:52,848 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting 12c9fed4ceb34602bb4b2c78e79553e8, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1732648549348 2024-11-26T19:15:52,848 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting cc13134c158940ce9742b4f24755e256, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1732648549343 2024-11-26T19:15:52,848 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting b41d94fff91f46b6997e09703e8ee195, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=44, earliestPutTs=1732648549426 2024-11-26T19:15:52,849 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting f55c1a2a88a14a7dbd27c6dc4a6f6498, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=44, earliestPutTs=1732648549426 2024-11-26T19:15:52,849 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting d86ff56850bb415c845fe379281a2266, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1732648551639 2024-11-26T19:15:52,849 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting 23a12e07036e4c2985e6151fc47168b6, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1732648551639 2024-11-26T19:15:52,858 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 741bfc4b4e2b320db573e476b103383f#B#compaction#365 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T19:15:52,858 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 741bfc4b4e2b320db573e476b103383f#A#compaction#366 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T19:15:52,858 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/B/2e3d25928fd1481282c3d8b220a04607 is 50, key is test_row_0/B:col10/1732648551639/Put/seqid=0 2024-11-26T19:15:52,858 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/A/2c11fcb559cf4f5da02cfe3ffd64953b is 50, key is test_row_0/A:col10/1732648551639/Put/seqid=0 2024-11-26T19:15:52,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742263_1439 (size=12104) 2024-11-26T19:15:52,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742264_1440 (size=12104) 2024-11-26T19:15:52,870 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/A/2c11fcb559cf4f5da02cfe3ffd64953b as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/A/2c11fcb559cf4f5da02cfe3ffd64953b 2024-11-26T19:15:52,871 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/B/2e3d25928fd1481282c3d8b220a04607 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/B/2e3d25928fd1481282c3d8b220a04607 2024-11-26T19:15:52,876 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 741bfc4b4e2b320db573e476b103383f/A of 741bfc4b4e2b320db573e476b103383f into 2c11fcb559cf4f5da02cfe3ffd64953b(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T19:15:52,876 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 741bfc4b4e2b320db573e476b103383f: 2024-11-26T19:15:52,876 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., storeName=741bfc4b4e2b320db573e476b103383f/A, priority=13, startTime=1732648552846; duration=0sec 2024-11-26T19:15:52,877 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T19:15:52,877 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 741bfc4b4e2b320db573e476b103383f:A 2024-11-26T19:15:52,877 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T19:15:52,878 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 741bfc4b4e2b320db573e476b103383f/B of 741bfc4b4e2b320db573e476b103383f into 2e3d25928fd1481282c3d8b220a04607(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T19:15:52,878 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 741bfc4b4e2b320db573e476b103383f: 2024-11-26T19:15:52,878 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., storeName=741bfc4b4e2b320db573e476b103383f/B, priority=13, startTime=1732648552846; duration=0sec 2024-11-26T19:15:52,878 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:15:52,878 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 741bfc4b4e2b320db573e476b103383f:B 2024-11-26T19:15:52,878 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T19:15:52,878 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1541): 741bfc4b4e2b320db573e476b103383f/C is initiating minor compaction (all files) 2024-11-26T19:15:52,878 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 741bfc4b4e2b320db573e476b103383f/C in TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. 2024-11-26T19:15:52,878 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/C/de6ca40c5e004b61a0dce3e14c1b8797, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/C/feb26fd9308a4719b36e217fc208430c, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/C/d179307cb34c4305a24c407e63432847] into tmpdir=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp, totalSize=32.9 K 2024-11-26T19:15:52,879 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting de6ca40c5e004b61a0dce3e14c1b8797, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1732648549348 2024-11-26T19:15:52,880 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting feb26fd9308a4719b36e217fc208430c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=44, earliestPutTs=1732648549426 2024-11-26T19:15:52,881 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting d179307cb34c4305a24c407e63432847, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1732648551639 2024-11-26T19:15:52,890 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 741bfc4b4e2b320db573e476b103383f#C#compaction#367 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T19:15:52,890 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/C/b359963e7f1045789fc2e91cfa41623b is 50, key is test_row_0/C:col10/1732648551639/Put/seqid=0 2024-11-26T19:15:52,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742265_1441 (size=12104) 2024-11-26T19:15:53,303 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/C/b359963e7f1045789fc2e91cfa41623b as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/C/b359963e7f1045789fc2e91cfa41623b 2024-11-26T19:15:53,307 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 741bfc4b4e2b320db573e476b103383f/C of 741bfc4b4e2b320db573e476b103383f into b359963e7f1045789fc2e91cfa41623b(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T19:15:53,307 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 741bfc4b4e2b320db573e476b103383f: 2024-11-26T19:15:53,307 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., storeName=741bfc4b4e2b320db573e476b103383f/C, priority=13, startTime=1732648552846; duration=0sec 2024-11-26T19:15:53,307 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:15:53,307 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 741bfc4b4e2b320db573e476b103383f:C 2024-11-26T19:15:53,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(8855): Flush requested on 741bfc4b4e2b320db573e476b103383f 2024-11-26T19:15:53,351 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 741bfc4b4e2b320db573e476b103383f 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-11-26T19:15:53,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 741bfc4b4e2b320db573e476b103383f, store=A 2024-11-26T19:15:53,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T19:15:53,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 741bfc4b4e2b320db573e476b103383f, store=B 2024-11-26T19:15:53,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T19:15:53,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 741bfc4b4e2b320db573e476b103383f, store=C 2024-11-26T19:15:53,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T19:15:53,356 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/A/b3b54c3af4e7486785d2c69746501409 is 50, key is test_row_0/A:col10/1732648552169/Put/seqid=0 2024-11-26T19:15:53,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742266_1442 (size=14341) 2024-11-26T19:15:53,396 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:53,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36730 deadline: 1732648613380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:53,396 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:53,396 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:53,396 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:53,396 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:53,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36770 deadline: 1732648613381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:53,397 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:53,397 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:53,397 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:53,511 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:53,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36730 deadline: 1732648613502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:53,511 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:53,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36770 deadline: 1732648613502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:53,512 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:53,512 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:53,512 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:53,512 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:53,512 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:53,512 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:53,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-26T19:15:53,522 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T19:15:53,523 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T19:15:53,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees 2024-11-26T19:15:53,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-26T19:15:53,525 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T19:15:53,525 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T19:15:53,525 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T19:15:53,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-26T19:15:53,677 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-26T19:15:53,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. 2024-11-26T19:15:53,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. as already flushing 2024-11-26T19:15:53,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. 2024-11-26T19:15:53,678 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:53,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:53,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:53,693 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:53,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36758 deadline: 1732648613681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:53,694 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to fe76acf732b2,42785,1732648423970 for 'test_row_2' in 741bfc4b4e2b320db573e476b103383f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4261 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T19:15:53,694 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:53,694 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:53,694 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:53,730 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:53,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36730 deadline: 1732648613722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:53,731 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:53,731 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:53,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36770 deadline: 1732648613722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:53,731 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:53,731 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:53,731 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:53,731 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:53,731 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:53,760 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/A/b3b54c3af4e7486785d2c69746501409 2024-11-26T19:15:53,767 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/B/f3a61c66dfee4bd3b4626addc0a9137b is 50, key is test_row_0/B:col10/1732648552169/Put/seqid=0 2024-11-26T19:15:53,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742267_1443 (size=12001) 2024-11-26T19:15:53,831 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-26T19:15:53,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. 2024-11-26T19:15:53,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. as already flushing 2024-11-26T19:15:53,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. 2024-11-26T19:15:53,831 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:53,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:53,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:53,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-26T19:15:53,846 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:53,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36744 deadline: 1732648613832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:53,846 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:53,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36712 deadline: 1732648613832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:53,846 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to fe76acf732b2,42785,1732648423970 for 'test_row_2' in 741bfc4b4e2b320db573e476b103383f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2214 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T19:15:53,847 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:53,847 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:53,847 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:53,847 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to fe76acf732b2,42785,1732648423970 for 'test_row_2' in 741bfc4b4e2b320db573e476b103383f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2206 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T19:15:53,847 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:53,847 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:53,847 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:53,983 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-26T19:15:53,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. 2024-11-26T19:15:53,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. as already flushing 2024-11-26T19:15:53,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. 2024-11-26T19:15:53,984 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:53,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:53,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:54,051 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:54,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36730 deadline: 1732648614042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:54,051 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:54,051 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:54,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36770 deadline: 1732648614042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:54,051 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:54,052 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:54,052 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:54,052 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:54,052 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:54,136 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-26T19:15:54,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. 2024-11-26T19:15:54,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. as already flushing 2024-11-26T19:15:54,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. 2024-11-26T19:15:54,137 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:54,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:54,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:54,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-26T19:15:54,172 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/B/f3a61c66dfee4bd3b4626addc0a9137b 2024-11-26T19:15:54,180 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/C/703b372c747e455a80390069c05a1f02 is 50, key is test_row_0/C:col10/1732648552169/Put/seqid=0 2024-11-26T19:15:54,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742268_1444 (size=12001) 2024-11-26T19:15:54,289 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-26T19:15:54,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. 2024-11-26T19:15:54,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. as already flushing 2024-11-26T19:15:54,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. 2024-11-26T19:15:54,290 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:54,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:54,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:54,442 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-26T19:15:54,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. 2024-11-26T19:15:54,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. as already flushing 2024-11-26T19:15:54,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. 2024-11-26T19:15:54,443 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:54,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:54,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:54,578 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:54,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36730 deadline: 1732648614562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:54,579 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:54,579 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:54,579 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:54,579 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:54,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36770 deadline: 1732648614562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:54,579 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:54,579 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:54,579 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:54,584 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/C/703b372c747e455a80390069c05a1f02 2024-11-26T19:15:54,588 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/A/b3b54c3af4e7486785d2c69746501409 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/A/b3b54c3af4e7486785d2c69746501409 2024-11-26T19:15:54,591 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/A/b3b54c3af4e7486785d2c69746501409, entries=200, sequenceid=93, filesize=14.0 K 2024-11-26T19:15:54,592 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/B/f3a61c66dfee4bd3b4626addc0a9137b as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/B/f3a61c66dfee4bd3b4626addc0a9137b 2024-11-26T19:15:54,595 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/B/f3a61c66dfee4bd3b4626addc0a9137b, entries=150, sequenceid=93, filesize=11.7 K 2024-11-26T19:15:54,595 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/C/703b372c747e455a80390069c05a1f02 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/C/703b372c747e455a80390069c05a1f02 2024-11-26T19:15:54,595 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-26T19:15:54,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. 2024-11-26T19:15:54,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. as already flushing 2024-11-26T19:15:54,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. 2024-11-26T19:15:54,596 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:54,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:54,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:54,600 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/C/703b372c747e455a80390069c05a1f02, entries=150, sequenceid=93, filesize=11.7 K 2024-11-26T19:15:54,600 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=46.96 KB/48090 for 741bfc4b4e2b320db573e476b103383f in 1250ms, sequenceid=93, compaction requested=false 2024-11-26T19:15:54,600 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 741bfc4b4e2b320db573e476b103383f: 2024-11-26T19:15:54,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-26T19:15:54,748 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-26T19:15:54,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. 2024-11-26T19:15:54,748 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2902): Flushing 741bfc4b4e2b320db573e476b103383f 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-26T19:15:54,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 741bfc4b4e2b320db573e476b103383f, store=A 2024-11-26T19:15:54,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:15:54,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 741bfc4b4e2b320db573e476b103383f, store=B 2024-11-26T19:15:54,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:15:54,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 741bfc4b4e2b320db573e476b103383f, store=C 2024-11-26T19:15:54,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:15:54,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/A/7d05fe7933c3443eb046fbe7f56a2bd6 is 50, key is test_row_0/A:col10/1732648553378/Put/seqid=0 2024-11-26T19:15:54,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742269_1445 (size=12001) 2024-11-26T19:15:55,158 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/A/7d05fe7933c3443eb046fbe7f56a2bd6 2024-11-26T19:15:55,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/B/52d3b2ae440f4c778452b92b9f2525c0 is 50, key is test_row_0/B:col10/1732648553378/Put/seqid=0 2024-11-26T19:15:55,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742270_1446 (size=12001) 2024-11-26T19:15:55,571 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/B/52d3b2ae440f4c778452b92b9f2525c0 2024-11-26T19:15:55,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/C/a8c8a13ad4e04895a36aa6b3441a7216 is 50, key is test_row_0/C:col10/1732648553378/Put/seqid=0 2024-11-26T19:15:55,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742271_1447 (size=12001) 2024-11-26T19:15:55,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(8855): Flush requested on 741bfc4b4e2b320db573e476b103383f 2024-11-26T19:15:55,594 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. as already flushing 2024-11-26T19:15:55,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-26T19:15:55,674 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732648555626, max timestamp=1732648555670Num uniques 859; 2024-11-26T19:15:55,674 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732648555626, max timestamp=1732648555670Num uniques 859; 2024-11-26T19:15:55,674 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-26T19:15:55,674 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-26T19:15:55,674 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732648555626, max timestamp=1732648555670Num uniques 859; 2024-11-26T19:15:55,674 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-26T19:15:55,772 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:55,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36730 deadline: 1732648615769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:55,773 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:55,773 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:55,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36770 deadline: 1732648615770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:55,773 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:55,773 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:55,773 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:55,773 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:55,773 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:55,870 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:55,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36712 deadline: 1732648615862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:55,870 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to fe76acf732b2,42785,1732648423970 for 'test_row_2' in 741bfc4b4e2b320db573e476b103383f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4230 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T19:15:55,871 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:55,871 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:55,871 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:55,882 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:55,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36744 deadline: 1732648615871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:55,883 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to fe76acf732b2,42785,1732648423970 for 'test_row_2' in 741bfc4b4e2b320db573e476b103383f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4250 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T19:15:55,883 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:55,883 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:55,883 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:55,894 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:55,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36730 deadline: 1732648615882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:55,894 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:55,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36770 deadline: 1732648615882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:55,894 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:55,894 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:55,895 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:55,895 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:55,895 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:55,895 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:55,984 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/C/a8c8a13ad4e04895a36aa6b3441a7216 2024-11-26T19:15:55,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/A/7d05fe7933c3443eb046fbe7f56a2bd6 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/A/7d05fe7933c3443eb046fbe7f56a2bd6 2024-11-26T19:15:55,991 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/A/7d05fe7933c3443eb046fbe7f56a2bd6, entries=150, sequenceid=103, filesize=11.7 K 2024-11-26T19:15:55,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/B/52d3b2ae440f4c778452b92b9f2525c0 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/B/52d3b2ae440f4c778452b92b9f2525c0 2024-11-26T19:15:55,995 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/B/52d3b2ae440f4c778452b92b9f2525c0, entries=150, sequenceid=103, filesize=11.7 K 2024-11-26T19:15:55,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/C/a8c8a13ad4e04895a36aa6b3441a7216 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/C/a8c8a13ad4e04895a36aa6b3441a7216 2024-11-26T19:15:55,999 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/C/a8c8a13ad4e04895a36aa6b3441a7216, entries=150, sequenceid=103, filesize=11.7 K 2024-11-26T19:15:56,000 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for 741bfc4b4e2b320db573e476b103383f in 1252ms, sequenceid=103, compaction requested=true 2024-11-26T19:15:56,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2603): Flush status journal for 741bfc4b4e2b320db573e476b103383f: 2024-11-26T19:15:56,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. 2024-11-26T19:15:56,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=118 2024-11-26T19:15:56,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster(4169): Remote procedure done, pid=118 2024-11-26T19:15:56,003 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-11-26T19:15:56,003 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4760 sec 2024-11-26T19:15:56,004 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees in 2.4800 sec 2024-11-26T19:15:56,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(8855): Flush requested on 741bfc4b4e2b320db573e476b103383f 2024-11-26T19:15:56,112 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 741bfc4b4e2b320db573e476b103383f 3/3 column families, dataSize=147.60 KB heapSize=420.84 KB 2024-11-26T19:15:56,114 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732648555749, max timestamp=1732648556109Num uniques 1068; 2024-11-26T19:15:56,114 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-26T19:15:56,114 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732648555749, max timestamp=1732648556109Num uniques 1068; 2024-11-26T19:15:56,114 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-26T19:15:56,114 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732648555749, max timestamp=1732648556109Num uniques 1068; 2024-11-26T19:15:56,115 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-26T19:15:56,115 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 741bfc4b4e2b320db573e476b103383f, store=A 2024-11-26T19:15:56,115 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T19:15:56,115 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 741bfc4b4e2b320db573e476b103383f, store=B 2024-11-26T19:15:56,115 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T19:15:56,115 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 741bfc4b4e2b320db573e476b103383f, store=C 2024-11-26T19:15:56,115 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T19:15:56,119 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/A/b51590d8309f467aa918f37bd8bf1174 is 50, key is test_row_0/A:col10/1732648555768/Put/seqid=0 2024-11-26T19:15:56,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742272_1448 (size=16931) 2024-11-26T19:15:56,218 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732648556179, max timestamp=1732648556215Num uniques 859; 2024-11-26T19:15:56,218 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-26T19:15:56,218 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732648556179, max timestamp=1732648556215Num uniques 859; 2024-11-26T19:15:56,219 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-26T19:15:56,219 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732648556179, max timestamp=1732648556215Num uniques 859; 2024-11-26T19:15:56,219 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-26T19:15:56,331 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732648556288, max timestamp=1732648556326Num uniques 1068; 2024-11-26T19:15:56,331 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-11-26T19:15:56,331 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732648556288, max timestamp=1732648556326Num uniques 1068; 2024-11-26T19:15:56,332 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-11-26T19:15:56,332 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732648556288, max timestamp=1732648556326Num uniques 1068; 2024-11-26T19:15:56,332 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-11-26T19:15:56,448 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1732648556418, max timestamp=1732648556443Num uniques 1077; 2024-11-26T19:15:56,448 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=8102, suffix heap size=139976,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=177 2024-11-26T19:15:56,449 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1732648556418, max timestamp=1732648556443Num uniques 1077; 2024-11-26T19:15:56,449 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=8102, suffix heap size=139976,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=177 2024-11-26T19:15:56,449 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1732648556418, max timestamp=1732648556443Num uniques 1077; 2024-11-26T19:15:56,449 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=8102, suffix heap size=139976,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=177 2024-11-26T19:15:56,529 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=142 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/A/b51590d8309f467aa918f37bd8bf1174 2024-11-26T19:15:56,537 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/B/ede4d0b8067c4c95aa70e3390af1258d is 50, key is test_row_0/B:col10/1732648555768/Put/seqid=0 2024-11-26T19:15:56,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742273_1449 (size=12151) 2024-11-26T19:15:56,541 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=142 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/B/ede4d0b8067c4c95aa70e3390af1258d 2024-11-26T19:15:56,547 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/C/8f26cce1c3ac48c1879f93283371e670 is 50, key is test_row_0/C:col10/1732648555768/Put/seqid=0 2024-11-26T19:15:56,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742274_1450 (size=12151) 2024-11-26T19:15:56,562 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1732648556518, max timestamp=1732648556557Num uniques 1036; 2024-11-26T19:15:56,562 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=10806, suffix heap size=135712,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=236 2024-11-26T19:15:56,562 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1732648556518, max timestamp=1732648556557Num uniques 1036; 2024-11-26T19:15:56,562 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=10806, suffix heap size=135712,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=236 2024-11-26T19:15:56,563 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1732648556518, max timestamp=1732648556557Num uniques 1036; 2024-11-26T19:15:56,563 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=10806, suffix heap size=135712,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=236 2024-11-26T19:15:56,674 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1732648556603, max timestamp=1732648556669Num uniques 1095; 2024-11-26T19:15:56,674 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=11220, suffix heap size=141848,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=245 2024-11-26T19:15:56,675 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1732648556603, max timestamp=1732648556669Num uniques 1095; 2024-11-26T19:15:56,675 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1732648556603, max timestamp=1732648556669Num uniques 1095; 2024-11-26T19:15:56,675 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=11220, suffix heap size=141848,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=245 2024-11-26T19:15:56,675 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=11220, suffix heap size=141848,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=245 2024-11-26T19:15:56,790 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1732648556764, max timestamp=1732648556785Num uniques 1104; 2024-11-26T19:15:56,790 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=9340, suffix heap size=142784,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=204 2024-11-26T19:15:56,790 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1732648556764, max timestamp=1732648556785Num uniques 1104; 2024-11-26T19:15:56,791 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1732648556764, max timestamp=1732648556785Num uniques 1104; 2024-11-26T19:15:56,791 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=9340, suffix heap size=142784,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=204 2024-11-26T19:15:56,791 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=9340, suffix heap size=142784,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=204 2024-11-26T19:15:56,915 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1732648556859, max timestamp=1732648556910Num uniques 1063; 2024-11-26T19:15:56,915 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1732648556859, max timestamp=1732648556910Num uniques 1063; 2024-11-26T19:15:56,915 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=9748, suffix heap size=138520,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=213 2024-11-26T19:15:56,915 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=9748, suffix heap size=138520,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=213 2024-11-26T19:15:56,915 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1732648556859, max timestamp=1732648556910Num uniques 1063; 2024-11-26T19:15:56,916 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=9748, suffix heap size=138520,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=213 2024-11-26T19:15:56,952 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=142 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/C/8f26cce1c3ac48c1879f93283371e670 2024-11-26T19:15:56,956 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/A/b51590d8309f467aa918f37bd8bf1174 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/A/b51590d8309f467aa918f37bd8bf1174 2024-11-26T19:15:56,959 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/A/b51590d8309f467aa918f37bd8bf1174, entries=250, sequenceid=142, filesize=16.5 K 2024-11-26T19:15:56,960 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/B/ede4d0b8067c4c95aa70e3390af1258d as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/B/ede4d0b8067c4c95aa70e3390af1258d 2024-11-26T19:15:56,963 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/B/ede4d0b8067c4c95aa70e3390af1258d, entries=150, sequenceid=142, filesize=11.9 K 2024-11-26T19:15:56,964 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/C/8f26cce1c3ac48c1879f93283371e670 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/C/8f26cce1c3ac48c1879f93283371e670 2024-11-26T19:15:56,967 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/C/8f26cce1c3ac48c1879f93283371e670, entries=150, sequenceid=142, filesize=11.9 K 2024-11-26T19:15:56,967 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~85.27 KB/87312, currentSize=93.93 KB/96180 for 741bfc4b4e2b320db573e476b103383f in 855ms, sequenceid=142, compaction requested=true 2024-11-26T19:15:56,967 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 741bfc4b4e2b320db573e476b103383f: 2024-11-26T19:15:56,968 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 741bfc4b4e2b320db573e476b103383f:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T19:15:56,968 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:15:56,968 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T19:15:56,968 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 741bfc4b4e2b320db573e476b103383f:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T19:15:56,968 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:15:56,968 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 741bfc4b4e2b320db573e476b103383f:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T19:15:56,968 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T19:15:56,968 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T19:15:56,969 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 55377 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T19:15:56,969 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48257 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T19:15:56,969 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1541): 741bfc4b4e2b320db573e476b103383f/A is initiating minor compaction (all files) 2024-11-26T19:15:56,969 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1541): 741bfc4b4e2b320db573e476b103383f/B is initiating minor compaction (all files) 2024-11-26T19:15:56,969 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 741bfc4b4e2b320db573e476b103383f/A in TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. 2024-11-26T19:15:56,969 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 741bfc4b4e2b320db573e476b103383f/B in TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. 2024-11-26T19:15:56,969 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/A/2c11fcb559cf4f5da02cfe3ffd64953b, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/A/b3b54c3af4e7486785d2c69746501409, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/A/7d05fe7933c3443eb046fbe7f56a2bd6, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/A/b51590d8309f467aa918f37bd8bf1174] into tmpdir=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp, totalSize=54.1 K 2024-11-26T19:15:56,969 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/B/2e3d25928fd1481282c3d8b220a04607, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/B/f3a61c66dfee4bd3b4626addc0a9137b, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/B/52d3b2ae440f4c778452b92b9f2525c0, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/B/ede4d0b8067c4c95aa70e3390af1258d] into tmpdir=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp, totalSize=47.1 K 2024-11-26T19:15:56,969 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2c11fcb559cf4f5da02cfe3ffd64953b, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1732648551639 2024-11-26T19:15:56,969 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting 2e3d25928fd1481282c3d8b220a04607, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1732648551639 2024-11-26T19:15:56,969 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting f3a61c66dfee4bd3b4626addc0a9137b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1732648552140 2024-11-26T19:15:56,969 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting b3b54c3af4e7486785d2c69746501409, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1732648552140 2024-11-26T19:15:56,970 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7d05fe7933c3443eb046fbe7f56a2bd6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1732648553366 2024-11-26T19:15:56,970 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting 52d3b2ae440f4c778452b92b9f2525c0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1732648553366 2024-11-26T19:15:56,970 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting ede4d0b8067c4c95aa70e3390af1258d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1732648555768 2024-11-26T19:15:56,970 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting b51590d8309f467aa918f37bd8bf1174, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1732648555749 2024-11-26T19:15:56,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(8855): Flush requested on 741bfc4b4e2b320db573e476b103383f 2024-11-26T19:15:56,973 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 741bfc4b4e2b320db573e476b103383f 3/3 column families, dataSize=100.63 KB heapSize=280.31 KB 2024-11-26T19:15:56,974 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 741bfc4b4e2b320db573e476b103383f, store=A 2024-11-26T19:15:56,974 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T19:15:56,974 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 741bfc4b4e2b320db573e476b103383f, store=B 2024-11-26T19:15:56,974 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T19:15:56,974 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 741bfc4b4e2b320db573e476b103383f, store=C 2024-11-26T19:15:56,974 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T19:15:56,979 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 741bfc4b4e2b320db573e476b103383f#A#compaction#377 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T19:15:56,979 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 741bfc4b4e2b320db573e476b103383f#B#compaction#378 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T19:15:56,979 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/A/57ef3666ef6842c7871c2a65c3237ba0 is 50, key is test_row_0/A:col10/1732648555768/Put/seqid=0 2024-11-26T19:15:56,979 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/B/4b8dac2818d8404c920ba4eac6e6537f is 50, key is test_row_0/B:col10/1732648555768/Put/seqid=0 2024-11-26T19:15:56,979 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/A/d3c4f2ba6b854fcdadcf12fdac421d6b is 50, key is test_row_0/A:col10/1732648556972/Put/seqid=0 2024-11-26T19:15:56,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742275_1451 (size=12391) 2024-11-26T19:15:56,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742276_1452 (size=12391) 2024-11-26T19:15:56,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742277_1453 (size=14741) 2024-11-26T19:15:56,990 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/A/d3c4f2ba6b854fcdadcf12fdac421d6b 2024-11-26T19:15:56,998 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/B/070c70f9219049f68c31223fab166ff5 is 50, key is test_row_0/B:col10/1732648556972/Put/seqid=0 2024-11-26T19:15:57,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742278_1454 (size=12301) 2024-11-26T19:15:57,055 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:57,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36770 deadline: 1732648617040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:57,055 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:57,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36730 deadline: 1732648617043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:57,056 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:57,056 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:57,056 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:57,056 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:57,056 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:57,056 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:57,171 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:57,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36770 deadline: 1732648617162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:57,172 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:57,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36730 deadline: 1732648617162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:57,172 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:57,172 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:57,172 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:57,172 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:57,172 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:57,172 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:57,390 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/B/4b8dac2818d8404c920ba4eac6e6537f as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/B/4b8dac2818d8404c920ba4eac6e6537f 2024-11-26T19:15:57,393 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:57,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36770 deadline: 1732648617382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:57,394 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:57,394 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:57,394 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:57,394 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:57,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36730 deadline: 1732648617382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:57,394 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 741bfc4b4e2b320db573e476b103383f/B of 741bfc4b4e2b320db573e476b103383f into 4b8dac2818d8404c920ba4eac6e6537f(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T19:15:57,394 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 741bfc4b4e2b320db573e476b103383f: 2024-11-26T19:15:57,394 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., storeName=741bfc4b4e2b320db573e476b103383f/B, priority=12, startTime=1732648556968; duration=0sec 2024-11-26T19:15:57,394 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T19:15:57,394 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 741bfc4b4e2b320db573e476b103383f:B 2024-11-26T19:15:57,395 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T19:15:57,400 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:57,400 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:57,400 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:57,401 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/A/57ef3666ef6842c7871c2a65c3237ba0 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/A/57ef3666ef6842c7871c2a65c3237ba0 2024-11-26T19:15:57,401 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48257 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T19:15:57,401 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1541): 741bfc4b4e2b320db573e476b103383f/C is initiating minor compaction (all files) 2024-11-26T19:15:57,401 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 741bfc4b4e2b320db573e476b103383f/C in TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. 2024-11-26T19:15:57,401 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/C/b359963e7f1045789fc2e91cfa41623b, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/C/703b372c747e455a80390069c05a1f02, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/C/a8c8a13ad4e04895a36aa6b3441a7216, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/C/8f26cce1c3ac48c1879f93283371e670] into tmpdir=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp, totalSize=47.1 K 2024-11-26T19:15:57,402 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting b359963e7f1045789fc2e91cfa41623b, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1732648551639 2024-11-26T19:15:57,402 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/B/070c70f9219049f68c31223fab166ff5 2024-11-26T19:15:57,402 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting 703b372c747e455a80390069c05a1f02, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1732648552140 2024-11-26T19:15:57,403 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting a8c8a13ad4e04895a36aa6b3441a7216, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1732648553366 2024-11-26T19:15:57,404 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting 8f26cce1c3ac48c1879f93283371e670, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1732648555768 2024-11-26T19:15:57,405 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 741bfc4b4e2b320db573e476b103383f/A of 741bfc4b4e2b320db573e476b103383f into 57ef3666ef6842c7871c2a65c3237ba0(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T19:15:57,405 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 741bfc4b4e2b320db573e476b103383f: 2024-11-26T19:15:57,405 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., storeName=741bfc4b4e2b320db573e476b103383f/A, priority=12, startTime=1732648556967; duration=0sec 2024-11-26T19:15:57,405 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:15:57,405 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 741bfc4b4e2b320db573e476b103383f:A 2024-11-26T19:15:57,411 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/C/b6d916657f044546aad705e55f69cb66 is 50, key is test_row_0/C:col10/1732648556972/Put/seqid=0 2024-11-26T19:15:57,416 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 741bfc4b4e2b320db573e476b103383f#C#compaction#382 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T19:15:57,417 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/C/2f8855040aa44190a8b22e9d3bd3757a is 50, key is test_row_0/C:col10/1732648555768/Put/seqid=0 2024-11-26T19:15:57,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742279_1455 (size=12301) 2024-11-26T19:15:57,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742280_1456 (size=12391) 2024-11-26T19:15:57,432 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/C/2f8855040aa44190a8b22e9d3bd3757a as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/C/2f8855040aa44190a8b22e9d3bd3757a 2024-11-26T19:15:57,435 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 741bfc4b4e2b320db573e476b103383f/C of 741bfc4b4e2b320db573e476b103383f into 2f8855040aa44190a8b22e9d3bd3757a(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T19:15:57,435 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 741bfc4b4e2b320db573e476b103383f: 2024-11-26T19:15:57,435 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., storeName=741bfc4b4e2b320db573e476b103383f/C, priority=12, startTime=1732648556968; duration=0sec 2024-11-26T19:15:57,436 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:15:57,436 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 741bfc4b4e2b320db573e476b103383f:C 2024-11-26T19:15:57,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-26T19:15:57,692 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T19:15:57,693 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T19:15:57,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees 2024-11-26T19:15:57,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-26T19:15:57,695 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T19:15:57,695 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T19:15:57,695 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T19:15:57,707 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:57,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36770 deadline: 1732648617701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:57,707 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:57,708 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:57,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36730 deadline: 1732648617701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:57,708 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:57,708 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:57,708 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:57,708 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:57,708 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:57,735 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:57,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36758 deadline: 1732648617732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:57,736 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to fe76acf732b2,42785,1732648423970 for 'test_row_2' in 741bfc4b4e2b320db573e476b103383f of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8304 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T19:15:57,736 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:57,737 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:57,737 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:57,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-26T19:15:57,823 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/C/b6d916657f044546aad705e55f69cb66 2024-11-26T19:15:57,827 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/A/d3c4f2ba6b854fcdadcf12fdac421d6b as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/A/d3c4f2ba6b854fcdadcf12fdac421d6b 2024-11-26T19:15:57,829 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/A/d3c4f2ba6b854fcdadcf12fdac421d6b, entries=200, sequenceid=276, filesize=14.4 K 2024-11-26T19:15:57,830 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/B/070c70f9219049f68c31223fab166ff5 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/B/070c70f9219049f68c31223fab166ff5 2024-11-26T19:15:57,833 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/B/070c70f9219049f68c31223fab166ff5, entries=150, sequenceid=276, filesize=12.0 K 2024-11-26T19:15:57,834 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/C/b6d916657f044546aad705e55f69cb66 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/C/b6d916657f044546aad705e55f69cb66 2024-11-26T19:15:57,837 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/C/b6d916657f044546aad705e55f69cb66, entries=150, sequenceid=276, filesize=12.0 K 2024-11-26T19:15:57,838 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~280.27 KB/286992, currentSize=80.51 KB/82440 for 741bfc4b4e2b320db573e476b103383f in 865ms, sequenceid=276, compaction requested=false 2024-11-26T19:15:57,838 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 741bfc4b4e2b320db573e476b103383f: 2024-11-26T19:15:57,848 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-26T19:15:57,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. 2024-11-26T19:15:57,848 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2902): Flushing 741bfc4b4e2b320db573e476b103383f 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-26T19:15:57,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 741bfc4b4e2b320db573e476b103383f, store=A 2024-11-26T19:15:57,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:15:57,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 741bfc4b4e2b320db573e476b103383f, store=B 2024-11-26T19:15:57,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:15:57,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 741bfc4b4e2b320db573e476b103383f, store=C 2024-11-26T19:15:57,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:15:57,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/A/5ce2cea0e1c547f983371f4f19c452a6 is 50, key is test_row_0/A:col10/1732648557005/Put/seqid=0 2024-11-26T19:15:57,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742281_1457 (size=12301) 2024-11-26T19:15:58,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-26T19:15:58,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(8855): Flush requested on 741bfc4b4e2b320db573e476b103383f 2024-11-26T19:15:58,227 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. as already flushing 2024-11-26T19:15:58,258 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=294 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/A/5ce2cea0e1c547f983371f4f19c452a6 2024-11-26T19:15:58,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/B/bb42897b36844730a4c0a3caf103b824 is 50, key is test_row_0/B:col10/1732648557005/Put/seqid=0 2024-11-26T19:15:58,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742282_1458 (size=12301) 2024-11-26T19:15:58,269 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=294 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/B/bb42897b36844730a4c0a3caf103b824 2024-11-26T19:15:58,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/C/2dab618ba5a24c889d1162a4903b9fde is 50, key is test_row_0/C:col10/1732648557005/Put/seqid=0 2024-11-26T19:15:58,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742283_1459 (size=12301) 2024-11-26T19:15:58,293 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:58,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36730 deadline: 1732648618292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:58,293 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:58,293 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:58,293 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:58,303 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:58,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36770 deadline: 1732648618293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:58,303 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:58,304 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:58,304 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:58,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-26T19:15:58,411 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:58,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36730 deadline: 1732648618402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:58,411 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:58,411 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:58,411 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:58,418 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:58,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36770 deadline: 1732648618412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:58,419 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:58,419 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:58,419 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:58,631 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:58,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36730 deadline: 1732648618621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:58,631 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:58,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36770 deadline: 1732648618622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:58,631 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:58,631 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:58,631 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:58,632 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:58,632 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:58,632 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:58,681 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=294 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/C/2dab618ba5a24c889d1162a4903b9fde 2024-11-26T19:15:58,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/A/5ce2cea0e1c547f983371f4f19c452a6 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/A/5ce2cea0e1c547f983371f4f19c452a6 2024-11-26T19:15:58,689 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/A/5ce2cea0e1c547f983371f4f19c452a6, entries=150, sequenceid=294, filesize=12.0 K 2024-11-26T19:15:58,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/B/bb42897b36844730a4c0a3caf103b824 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/B/bb42897b36844730a4c0a3caf103b824 2024-11-26T19:15:58,693 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/B/bb42897b36844730a4c0a3caf103b824, entries=150, sequenceid=294, filesize=12.0 K 2024-11-26T19:15:58,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/C/2dab618ba5a24c889d1162a4903b9fde as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/C/2dab618ba5a24c889d1162a4903b9fde 2024-11-26T19:15:58,696 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/C/2dab618ba5a24c889d1162a4903b9fde, entries=150, sequenceid=294, filesize=12.0 K 2024-11-26T19:15:58,697 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=100.63 KB/103050 for 741bfc4b4e2b320db573e476b103383f in 849ms, sequenceid=294, compaction requested=true 2024-11-26T19:15:58,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2603): Flush status journal for 741bfc4b4e2b320db573e476b103383f: 2024-11-26T19:15:58,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. 2024-11-26T19:15:58,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=120 2024-11-26T19:15:58,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster(4169): Remote procedure done, pid=120 2024-11-26T19:15:58,700 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-11-26T19:15:58,700 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0040 sec 2024-11-26T19:15:58,702 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees in 1.0080 sec 2024-11-26T19:15:58,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-26T19:15:58,832 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T19:15:58,833 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T19:15:58,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees 2024-11-26T19:15:58,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-26T19:15:58,834 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T19:15:58,835 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T19:15:58,835 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T19:15:58,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-26T19:15:58,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(8855): Flush requested on 741bfc4b4e2b320db573e476b103383f 2024-11-26T19:15:58,949 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 741bfc4b4e2b320db573e476b103383f 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-26T19:15:58,950 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 741bfc4b4e2b320db573e476b103383f, store=A 2024-11-26T19:15:58,950 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:15:58,950 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 741bfc4b4e2b320db573e476b103383f, store=B 2024-11-26T19:15:58,950 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:15:58,950 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 741bfc4b4e2b320db573e476b103383f, store=C 2024-11-26T19:15:58,950 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:15:58,954 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/A/4628ad5741c74632957a7a75dae34a05 is 50, key is test_row_0/A:col10/1732648558291/Put/seqid=0 2024-11-26T19:15:58,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742284_1460 (size=17181) 2024-11-26T19:15:58,987 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-26T19:15:58,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. 2024-11-26T19:15:58,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. as already flushing 2024-11-26T19:15:58,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. 2024-11-26T19:15:58,987 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:58,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:58,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:58,992 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:58,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36770 deadline: 1732648618981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:58,992 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:58,992 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:58,992 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:58,992 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:58,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36730 deadline: 1732648618986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:58,993 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:58,993 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:58,993 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:59,112 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:59,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36770 deadline: 1732648619101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:59,112 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:59,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36730 deadline: 1732648619101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:59,112 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:59,112 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:59,112 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:59,112 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:59,112 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:59,112 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:59,139 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-26T19:15:59,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. 2024-11-26T19:15:59,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. as already flushing 2024-11-26T19:15:59,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. 2024-11-26T19:15:59,140 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:59,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:59,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:59,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-26T19:15:59,293 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-26T19:15:59,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. 2024-11-26T19:15:59,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. as already flushing 2024-11-26T19:15:59,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. 2024-11-26T19:15:59,293 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:59,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:59,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:59,331 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:59,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36770 deadline: 1732648619321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:59,332 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:59,332 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:59,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36730 deadline: 1732648619321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:59,332 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:59,332 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:59,332 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:59,332 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:59,332 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:59,358 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=314 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/A/4628ad5741c74632957a7a75dae34a05 2024-11-26T19:15:59,365 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/B/1f75d62a2e074cbb87c33cb26fc32de0 is 50, key is test_row_0/B:col10/1732648558291/Put/seqid=0 2024-11-26T19:15:59,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742285_1461 (size=12301) 2024-11-26T19:15:59,445 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-26T19:15:59,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. 2024-11-26T19:15:59,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. as already flushing 2024-11-26T19:15:59,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. 2024-11-26T19:15:59,446 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:59,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:59,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:59,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-26T19:15:59,598 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-26T19:15:59,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. 2024-11-26T19:15:59,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. as already flushing 2024-11-26T19:15:59,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. 2024-11-26T19:15:59,599 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:59,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:59,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:59,648 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:59,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36770 deadline: 1732648619642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:59,648 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:59,648 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:59,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36730 deadline: 1732648619642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:59,648 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:59,648 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:59,648 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:59,648 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:59,648 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:59,750 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-26T19:15:59,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. 2024-11-26T19:15:59,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. as already flushing 2024-11-26T19:15:59,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. 2024-11-26T19:15:59,751 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:59,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:59,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:59,772 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=314 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/B/1f75d62a2e074cbb87c33cb26fc32de0 2024-11-26T19:15:59,779 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/C/2bcc29443d4d42d9b3d43efafc67c907 is 50, key is test_row_0/C:col10/1732648558291/Put/seqid=0 2024-11-26T19:15:59,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742286_1462 (size=12301) 2024-11-26T19:15:59,899 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:59,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36744 deadline: 1732648619892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:59,899 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to fe76acf732b2,42785,1732648423970 for 'test_row_2' in 741bfc4b4e2b320db573e476b103383f of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8266 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T19:15:59,900 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:59,900 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:59,900 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:59,903 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-26T19:15:59,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. 2024-11-26T19:15:59,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. as already flushing 2024-11-26T19:15:59,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. 2024-11-26T19:15:59,903 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:59,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:59,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:15:59,912 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:15:59,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36712 deadline: 1732648619902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:15:59,913 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to fe76acf732b2,42785,1732648423970 for 'test_row_2' in 741bfc4b4e2b320db573e476b103383f of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8272 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T19:15:59,913 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:59,913 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:15:59,913 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:15:59,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-26T19:16:00,056 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-26T19:16:00,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. 2024-11-26T19:16:00,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. as already flushing 2024-11-26T19:16:00,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. 2024-11-26T19:16:00,056 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:16:00,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:16:00,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:16:00,162 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:00,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36730 deadline: 1732648620152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:00,162 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:00,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36770 deadline: 1732648620161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:00,163 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:00,163 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:00,163 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:16:00,163 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:00,163 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:00,163 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:16:00,183 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=314 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/C/2bcc29443d4d42d9b3d43efafc67c907 2024-11-26T19:16:00,187 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/A/4628ad5741c74632957a7a75dae34a05 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/A/4628ad5741c74632957a7a75dae34a05 2024-11-26T19:16:00,191 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/A/4628ad5741c74632957a7a75dae34a05, entries=250, sequenceid=314, filesize=16.8 K 2024-11-26T19:16:00,191 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/B/1f75d62a2e074cbb87c33cb26fc32de0 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/B/1f75d62a2e074cbb87c33cb26fc32de0 2024-11-26T19:16:00,194 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/B/1f75d62a2e074cbb87c33cb26fc32de0, entries=150, sequenceid=314, filesize=12.0 K 2024-11-26T19:16:00,195 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/C/2bcc29443d4d42d9b3d43efafc67c907 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/C/2bcc29443d4d42d9b3d43efafc67c907 2024-11-26T19:16:00,199 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/C/2bcc29443d4d42d9b3d43efafc67c907, entries=150, sequenceid=314, filesize=12.0 K 2024-11-26T19:16:00,200 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 741bfc4b4e2b320db573e476b103383f in 1251ms, sequenceid=314, compaction requested=true 2024-11-26T19:16:00,200 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 741bfc4b4e2b320db573e476b103383f: 2024-11-26T19:16:00,200 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 741bfc4b4e2b320db573e476b103383f:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T19:16:00,200 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:16:00,200 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T19:16:00,200 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 741bfc4b4e2b320db573e476b103383f:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T19:16:00,200 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:16:00,200 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 741bfc4b4e2b320db573e476b103383f:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T19:16:00,200 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T19:16:00,200 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T19:16:00,201 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 56614 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T19:16:00,201 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1541): 741bfc4b4e2b320db573e476b103383f/A is initiating minor compaction (all files) 2024-11-26T19:16:00,201 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49294 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T19:16:00,201 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 741bfc4b4e2b320db573e476b103383f/A in TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. 2024-11-26T19:16:00,201 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1541): 741bfc4b4e2b320db573e476b103383f/B is initiating minor compaction (all files) 2024-11-26T19:16:00,201 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 741bfc4b4e2b320db573e476b103383f/B in TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. 2024-11-26T19:16:00,202 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/A/57ef3666ef6842c7871c2a65c3237ba0, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/A/d3c4f2ba6b854fcdadcf12fdac421d6b, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/A/5ce2cea0e1c547f983371f4f19c452a6, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/A/4628ad5741c74632957a7a75dae34a05] into tmpdir=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp, totalSize=55.3 K 2024-11-26T19:16:00,202 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/B/4b8dac2818d8404c920ba4eac6e6537f, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/B/070c70f9219049f68c31223fab166ff5, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/B/bb42897b36844730a4c0a3caf103b824, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/B/1f75d62a2e074cbb87c33cb26fc32de0] into tmpdir=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp, totalSize=48.1 K 2024-11-26T19:16:00,202 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting 57ef3666ef6842c7871c2a65c3237ba0, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1732648555768 2024-11-26T19:16:00,202 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting 4b8dac2818d8404c920ba4eac6e6537f, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1732648555768 2024-11-26T19:16:00,202 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting d3c4f2ba6b854fcdadcf12fdac421d6b, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1732648556931 2024-11-26T19:16:00,202 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting 070c70f9219049f68c31223fab166ff5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1732648556959 2024-11-26T19:16:00,202 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5ce2cea0e1c547f983371f4f19c452a6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1732648557005 2024-11-26T19:16:00,202 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting bb42897b36844730a4c0a3caf103b824, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1732648557005 2024-11-26T19:16:00,203 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4628ad5741c74632957a7a75dae34a05, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1732648558227 2024-11-26T19:16:00,203 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting 1f75d62a2e074cbb87c33cb26fc32de0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1732648558276 2024-11-26T19:16:00,209 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-26T19:16:00,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. 2024-11-26T19:16:00,209 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2902): Flushing 741bfc4b4e2b320db573e476b103383f 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-26T19:16:00,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 741bfc4b4e2b320db573e476b103383f, store=A 2024-11-26T19:16:00,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:16:00,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 741bfc4b4e2b320db573e476b103383f, store=B 2024-11-26T19:16:00,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:16:00,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 741bfc4b4e2b320db573e476b103383f, store=C 2024-11-26T19:16:00,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:16:00,213 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 741bfc4b4e2b320db573e476b103383f#B#compaction#390 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T19:16:00,213 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 741bfc4b4e2b320db573e476b103383f#A#compaction#389 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T19:16:00,214 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/B/8463b575eec544c484e0ca2e5b672b70 is 50, key is test_row_0/B:col10/1732648558291/Put/seqid=0 2024-11-26T19:16:00,214 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/A/505bd621263f44f7895e792df03316ce is 50, key is test_row_0/A:col10/1732648558291/Put/seqid=0 2024-11-26T19:16:00,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/A/da0d7b2a5baf47af93c8ec99a4df5dbe is 50, key is test_row_0/A:col10/1732648558985/Put/seqid=0 2024-11-26T19:16:00,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742287_1463 (size=12677) 2024-11-26T19:16:00,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742288_1464 (size=12677) 2024-11-26T19:16:00,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742289_1465 (size=9857) 2024-11-26T19:16:00,224 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/A/da0d7b2a5baf47af93c8ec99a4df5dbe 2024-11-26T19:16:00,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/B/af68b1e0375040e991f20f0372b67db1 is 50, key is test_row_0/B:col10/1732648558985/Put/seqid=0 2024-11-26T19:16:00,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742290_1466 (size=9857) 2024-11-26T19:16:00,626 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/B/8463b575eec544c484e0ca2e5b672b70 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/B/8463b575eec544c484e0ca2e5b672b70 2024-11-26T19:16:00,626 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/A/505bd621263f44f7895e792df03316ce as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/A/505bd621263f44f7895e792df03316ce 2024-11-26T19:16:00,635 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 741bfc4b4e2b320db573e476b103383f/A of 741bfc4b4e2b320db573e476b103383f into 505bd621263f44f7895e792df03316ce(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T19:16:00,635 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/B/af68b1e0375040e991f20f0372b67db1 2024-11-26T19:16:00,635 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 741bfc4b4e2b320db573e476b103383f: 2024-11-26T19:16:00,635 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., storeName=741bfc4b4e2b320db573e476b103383f/A, priority=12, startTime=1732648560200; duration=0sec 2024-11-26T19:16:00,636 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T19:16:00,636 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 741bfc4b4e2b320db573e476b103383f:A 2024-11-26T19:16:00,636 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T19:16:00,638 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 741bfc4b4e2b320db573e476b103383f/B of 741bfc4b4e2b320db573e476b103383f into 8463b575eec544c484e0ca2e5b672b70(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T19:16:00,638 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 741bfc4b4e2b320db573e476b103383f: 2024-11-26T19:16:00,638 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., storeName=741bfc4b4e2b320db573e476b103383f/B, priority=12, startTime=1732648560200; duration=0sec 2024-11-26T19:16:00,638 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:16:00,638 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 741bfc4b4e2b320db573e476b103383f:B 2024-11-26T19:16:00,640 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49294 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T19:16:00,642 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1541): 741bfc4b4e2b320db573e476b103383f/C is initiating minor compaction (all files) 2024-11-26T19:16:00,642 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 741bfc4b4e2b320db573e476b103383f/C in TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. 2024-11-26T19:16:00,642 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/C/2f8855040aa44190a8b22e9d3bd3757a, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/C/b6d916657f044546aad705e55f69cb66, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/C/2dab618ba5a24c889d1162a4903b9fde, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/C/2bcc29443d4d42d9b3d43efafc67c907] into tmpdir=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp, totalSize=48.1 K 2024-11-26T19:16:00,646 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2f8855040aa44190a8b22e9d3bd3757a, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1732648555768 2024-11-26T19:16:00,646 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting b6d916657f044546aad705e55f69cb66, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1732648556959 2024-11-26T19:16:00,647 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2dab618ba5a24c889d1162a4903b9fde, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1732648557005 2024-11-26T19:16:00,647 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2bcc29443d4d42d9b3d43efafc67c907, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1732648558276 2024-11-26T19:16:00,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/C/8a4a90c8568a4137a69feb1f4566e551 is 50, key is test_row_0/C:col10/1732648558985/Put/seqid=0 2024-11-26T19:16:00,662 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 741bfc4b4e2b320db573e476b103383f#C#compaction#394 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T19:16:00,662 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/C/ca0549ea4d934b2ab9974943daf6d0c7 is 50, key is test_row_0/C:col10/1732648558291/Put/seqid=0 2024-11-26T19:16:00,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742291_1467 (size=9857) 2024-11-26T19:16:00,691 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/C/8a4a90c8568a4137a69feb1f4566e551 2024-11-26T19:16:00,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742292_1468 (size=12677) 2024-11-26T19:16:00,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/A/da0d7b2a5baf47af93c8ec99a4df5dbe as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/A/da0d7b2a5baf47af93c8ec99a4df5dbe 2024-11-26T19:16:00,699 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/A/da0d7b2a5baf47af93c8ec99a4df5dbe, entries=100, sequenceid=326, filesize=9.6 K 2024-11-26T19:16:00,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/B/af68b1e0375040e991f20f0372b67db1 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/B/af68b1e0375040e991f20f0372b67db1 2024-11-26T19:16:00,705 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/B/af68b1e0375040e991f20f0372b67db1, entries=100, sequenceid=326, filesize=9.6 K 2024-11-26T19:16:00,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/C/8a4a90c8568a4137a69feb1f4566e551 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/C/8a4a90c8568a4137a69feb1f4566e551 2024-11-26T19:16:00,710 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/C/8a4a90c8568a4137a69feb1f4566e551, entries=100, sequenceid=326, filesize=9.6 K 2024-11-26T19:16:00,711 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=0 B/0 for 741bfc4b4e2b320db573e476b103383f in 502ms, sequenceid=326, compaction requested=false 2024-11-26T19:16:00,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2603): Flush status journal for 741bfc4b4e2b320db573e476b103383f: 2024-11-26T19:16:00,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. 2024-11-26T19:16:00,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-11-26T19:16:00,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster(4169): Remote procedure done, pid=122 2024-11-26T19:16:00,715 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-11-26T19:16:00,715 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8780 sec 2024-11-26T19:16:00,716 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees in 1.8820 sec 2024-11-26T19:16:00,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-26T19:16:00,982 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T19:16:00,983 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T19:16:00,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-11-26T19:16:00,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-26T19:16:00,985 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T19:16:00,985 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T19:16:00,985 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T19:16:01,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-26T19:16:01,100 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/C/ca0549ea4d934b2ab9974943daf6d0c7 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/C/ca0549ea4d934b2ab9974943daf6d0c7 2024-11-26T19:16:01,104 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 741bfc4b4e2b320db573e476b103383f/C of 741bfc4b4e2b320db573e476b103383f into ca0549ea4d934b2ab9974943daf6d0c7(size=12.4 K), total size for store is 22.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T19:16:01,104 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 741bfc4b4e2b320db573e476b103383f: 2024-11-26T19:16:01,104 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., storeName=741bfc4b4e2b320db573e476b103383f/C, priority=12, startTime=1732648560200; duration=0sec 2024-11-26T19:16:01,105 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:16:01,105 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 741bfc4b4e2b320db573e476b103383f:C 2024-11-26T19:16:01,138 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-26T19:16:01,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. 2024-11-26T19:16:01,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for 741bfc4b4e2b320db573e476b103383f: 2024-11-26T19:16:01,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. 2024-11-26T19:16:01,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-11-26T19:16:01,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-11-26T19:16:01,142 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-11-26T19:16:01,142 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 155 msec 2024-11-26T19:16:01,143 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 159 msec 2024-11-26T19:16:01,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(8855): Flush requested on 741bfc4b4e2b320db573e476b103383f 2024-11-26T19:16:01,184 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 741bfc4b4e2b320db573e476b103383f 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-26T19:16:01,185 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 741bfc4b4e2b320db573e476b103383f, store=A 2024-11-26T19:16:01,185 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:16:01,185 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 741bfc4b4e2b320db573e476b103383f, store=B 2024-11-26T19:16:01,185 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:16:01,185 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 741bfc4b4e2b320db573e476b103383f, store=C 2024-11-26T19:16:01,185 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:16:01,189 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/A/3deed8cf987f46ee94d52b29f05ffd1f is 50, key is test_row_0/A:col10/1732648561183/Put/seqid=0 2024-11-26T19:16:01,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742293_1469 (size=14737) 2024-11-26T19:16:01,193 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=340 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/A/3deed8cf987f46ee94d52b29f05ffd1f 2024-11-26T19:16:01,199 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/B/edaa78381ad947c7babd1d5e0c360134 is 50, key is test_row_0/B:col10/1732648561183/Put/seqid=0 2024-11-26T19:16:01,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742294_1470 (size=9857) 2024-11-26T19:16:01,260 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732648561197, max timestamp=1732648561257Num uniques 859; 2024-11-26T19:16:01,260 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-26T19:16:01,261 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732648561197, max timestamp=1732648561257Num uniques 859; 2024-11-26T19:16:01,261 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732648561197, max timestamp=1732648561257Num uniques 859; 2024-11-26T19:16:01,261 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-26T19:16:01,261 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-26T19:16:01,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-26T19:16:01,302 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T19:16:01,302 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T19:16:01,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-11-26T19:16:01,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-26T19:16:01,304 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T19:16:01,305 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T19:16:01,305 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T19:16:01,345 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:01,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36770 deadline: 1732648621337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:01,345 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:01,345 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:01,345 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:16:01,352 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:01,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36730 deadline: 1732648621338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:01,352 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:01,352 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:01,352 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:16:01,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-26T19:16:01,457 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-26T19:16:01,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. 2024-11-26T19:16:01,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. as already flushing 2024-11-26T19:16:01,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. 2024-11-26T19:16:01,458 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:16:01,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:16:01,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:16:01,463 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:01,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36770 deadline: 1732648621452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:01,463 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:01,463 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:01,463 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:16:01,467 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:01,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36730 deadline: 1732648621462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:01,467 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:01,467 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:01,467 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:16:01,609 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=340 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/B/edaa78381ad947c7babd1d5e0c360134 2024-11-26T19:16:01,609 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-26T19:16:01,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. 2024-11-26T19:16:01,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. as already flushing 2024-11-26T19:16:01,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. 2024-11-26T19:16:01,610 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:16:01,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:16:01,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:16:01,616 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/C/5da694f9600d4813bb787d9cb753b75c is 50, key is test_row_0/C:col10/1732648561183/Put/seqid=0 2024-11-26T19:16:01,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742295_1471 (size=9857) 2024-11-26T19:16:01,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-26T19:16:01,680 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:01,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36770 deadline: 1732648621672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:01,680 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:01,680 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:01,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36730 deadline: 1732648621672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:01,680 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:01,680 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:16:01,681 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:01,681 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:01,681 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:16:01,762 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-26T19:16:01,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. 2024-11-26T19:16:01,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. as already flushing 2024-11-26T19:16:01,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. 2024-11-26T19:16:01,763 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:16:01,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:16:01,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:16:01,914 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-26T19:16:01,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. 2024-11-26T19:16:01,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. as already flushing 2024-11-26T19:16:01,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. 2024-11-26T19:16:01,915 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:16:01,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:16:01,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:16:01,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-26T19:16:01,994 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:01,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36770 deadline: 1732648621982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:01,994 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:01,994 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:01,994 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:16:02,000 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:02,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36730 deadline: 1732648621991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:02,001 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:02,001 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:02,001 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:16:02,020 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=340 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/C/5da694f9600d4813bb787d9cb753b75c 2024-11-26T19:16:02,024 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/A/3deed8cf987f46ee94d52b29f05ffd1f as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/A/3deed8cf987f46ee94d52b29f05ffd1f 2024-11-26T19:16:02,027 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/A/3deed8cf987f46ee94d52b29f05ffd1f, entries=200, sequenceid=340, filesize=14.4 K 2024-11-26T19:16:02,028 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/B/edaa78381ad947c7babd1d5e0c360134 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/B/edaa78381ad947c7babd1d5e0c360134 2024-11-26T19:16:02,031 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/B/edaa78381ad947c7babd1d5e0c360134, entries=100, sequenceid=340, filesize=9.6 K 2024-11-26T19:16:02,031 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/C/5da694f9600d4813bb787d9cb753b75c as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/C/5da694f9600d4813bb787d9cb753b75c 2024-11-26T19:16:02,035 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/C/5da694f9600d4813bb787d9cb753b75c, entries=100, sequenceid=340, filesize=9.6 K 2024-11-26T19:16:02,035 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 741bfc4b4e2b320db573e476b103383f in 851ms, sequenceid=340, compaction requested=true 2024-11-26T19:16:02,035 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 741bfc4b4e2b320db573e476b103383f: 2024-11-26T19:16:02,036 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 741bfc4b4e2b320db573e476b103383f:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T19:16:02,036 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:16:02,036 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T19:16:02,036 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 741bfc4b4e2b320db573e476b103383f:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T19:16:02,036 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:16:02,036 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T19:16:02,036 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 741bfc4b4e2b320db573e476b103383f:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T19:16:02,036 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T19:16:02,036 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 32391 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T19:16:02,036 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37271 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T19:16:02,037 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1541): 741bfc4b4e2b320db573e476b103383f/B is initiating minor compaction (all files) 2024-11-26T19:16:02,037 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1541): 741bfc4b4e2b320db573e476b103383f/A is initiating minor compaction (all files) 2024-11-26T19:16:02,037 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 741bfc4b4e2b320db573e476b103383f/B in TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. 2024-11-26T19:16:02,037 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 741bfc4b4e2b320db573e476b103383f/A in TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. 2024-11-26T19:16:02,037 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/B/8463b575eec544c484e0ca2e5b672b70, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/B/af68b1e0375040e991f20f0372b67db1, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/B/edaa78381ad947c7babd1d5e0c360134] into tmpdir=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp, totalSize=31.6 K 2024-11-26T19:16:02,037 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/A/505bd621263f44f7895e792df03316ce, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/A/da0d7b2a5baf47af93c8ec99a4df5dbe, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/A/3deed8cf987f46ee94d52b29f05ffd1f] into tmpdir=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp, totalSize=36.4 K 2024-11-26T19:16:02,037 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting 8463b575eec544c484e0ca2e5b672b70, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1732648558276 2024-11-26T19:16:02,037 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting 505bd621263f44f7895e792df03316ce, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1732648558276 2024-11-26T19:16:02,037 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting af68b1e0375040e991f20f0372b67db1, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1732648558980 2024-11-26T19:16:02,037 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting da0d7b2a5baf47af93c8ec99a4df5dbe, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1732648558980 2024-11-26T19:16:02,037 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting edaa78381ad947c7babd1d5e0c360134, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1732648561180 2024-11-26T19:16:02,038 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3deed8cf987f46ee94d52b29f05ffd1f, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1732648561178 2024-11-26T19:16:02,043 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 741bfc4b4e2b320db573e476b103383f#A#compaction#398 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T19:16:02,043 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 741bfc4b4e2b320db573e476b103383f#B#compaction#399 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T19:16:02,044 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/A/d5569930f78f4ef4865c15d78bb28feb is 50, key is test_row_0/A:col10/1732648561183/Put/seqid=0 2024-11-26T19:16:02,044 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/B/1705406432154a82a6d671962c9d71a9 is 50, key is test_row_0/B:col10/1732648561183/Put/seqid=0 2024-11-26T19:16:02,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742296_1472 (size=12779) 2024-11-26T19:16:02,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742297_1473 (size=12779) 2024-11-26T19:16:02,064 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/A/d5569930f78f4ef4865c15d78bb28feb as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/A/d5569930f78f4ef4865c15d78bb28feb 2024-11-26T19:16:02,066 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-26T19:16:02,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. 2024-11-26T19:16:02,067 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing 741bfc4b4e2b320db573e476b103383f 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-26T19:16:02,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 741bfc4b4e2b320db573e476b103383f, store=A 2024-11-26T19:16:02,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T19:16:02,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 741bfc4b4e2b320db573e476b103383f, store=B 2024-11-26T19:16:02,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T19:16:02,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 741bfc4b4e2b320db573e476b103383f, store=C 2024-11-26T19:16:02,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T19:16:02,069 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 741bfc4b4e2b320db573e476b103383f/A of 741bfc4b4e2b320db573e476b103383f into d5569930f78f4ef4865c15d78bb28feb(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T19:16:02,069 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 741bfc4b4e2b320db573e476b103383f: 2024-11-26T19:16:02,069 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., storeName=741bfc4b4e2b320db573e476b103383f/A, priority=13, startTime=1732648562036; duration=0sec 2024-11-26T19:16:02,069 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T19:16:02,069 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 741bfc4b4e2b320db573e476b103383f:A 2024-11-26T19:16:02,070 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T19:16:02,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/A/51215cb7cdf34728843fa1c4dd0a5574 is 50, key is test_row_0/A:col10/1732648561335/Put/seqid=0 2024-11-26T19:16:02,073 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 32391 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T19:16:02,073 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1541): 741bfc4b4e2b320db573e476b103383f/C is initiating minor compaction (all files) 2024-11-26T19:16:02,073 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 741bfc4b4e2b320db573e476b103383f/C in TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. 2024-11-26T19:16:02,073 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/C/ca0549ea4d934b2ab9974943daf6d0c7, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/C/8a4a90c8568a4137a69feb1f4566e551, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/C/5da694f9600d4813bb787d9cb753b75c] into tmpdir=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp, totalSize=31.6 K 2024-11-26T19:16:02,075 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting ca0549ea4d934b2ab9974943daf6d0c7, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1732648558276 2024-11-26T19:16:02,076 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8a4a90c8568a4137a69feb1f4566e551, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1732648558980 2024-11-26T19:16:02,076 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5da694f9600d4813bb787d9cb753b75c, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1732648561180 2024-11-26T19:16:02,087 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 741bfc4b4e2b320db573e476b103383f#C#compaction#401 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T19:16:02,088 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/C/49a99ab4394045bc857f9e20ac394ae6 is 50, key is test_row_0/C:col10/1732648561183/Put/seqid=0 2024-11-26T19:16:02,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742298_1474 (size=12301) 2024-11-26T19:16:02,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742299_1475 (size=12779) 2024-11-26T19:16:02,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-26T19:16:02,462 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/B/1705406432154a82a6d671962c9d71a9 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/B/1705406432154a82a6d671962c9d71a9 2024-11-26T19:16:02,466 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 741bfc4b4e2b320db573e476b103383f/B of 741bfc4b4e2b320db573e476b103383f into 1705406432154a82a6d671962c9d71a9(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T19:16:02,466 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 741bfc4b4e2b320db573e476b103383f: 2024-11-26T19:16:02,467 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., storeName=741bfc4b4e2b320db573e476b103383f/B, priority=13, startTime=1732648562036; duration=0sec 2024-11-26T19:16:02,467 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:16:02,467 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 741bfc4b4e2b320db573e476b103383f:B 2024-11-26T19:16:02,490 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=376 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/A/51215cb7cdf34728843fa1c4dd0a5574 2024-11-26T19:16:02,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/B/d09ffbdba8f1471783d06d0177318ad5 is 50, key is test_row_0/B:col10/1732648561335/Put/seqid=0 2024-11-26T19:16:02,501 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/C/49a99ab4394045bc857f9e20ac394ae6 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/C/49a99ab4394045bc857f9e20ac394ae6 2024-11-26T19:16:02,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742300_1476 (size=12301) 2024-11-26T19:16:02,504 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=376 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/B/d09ffbdba8f1471783d06d0177318ad5 2024-11-26T19:16:02,506 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 741bfc4b4e2b320db573e476b103383f/C of 741bfc4b4e2b320db573e476b103383f into 49a99ab4394045bc857f9e20ac394ae6(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T19:16:02,506 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 741bfc4b4e2b320db573e476b103383f: 2024-11-26T19:16:02,506 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., storeName=741bfc4b4e2b320db573e476b103383f/C, priority=13, startTime=1732648562036; duration=0sec 2024-11-26T19:16:02,506 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:16:02,506 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 741bfc4b4e2b320db573e476b103383f:C 2024-11-26T19:16:02,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(8855): Flush requested on 741bfc4b4e2b320db573e476b103383f 2024-11-26T19:16:02,508 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. as already flushing 2024-11-26T19:16:02,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/C/2595a75695074136a78f01b230e4891d is 50, key is test_row_0/C:col10/1732648561335/Put/seqid=0 2024-11-26T19:16:02,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742301_1477 (size=12301) 2024-11-26T19:16:02,541 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:02,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36770 deadline: 1732648622538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:02,542 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:02,542 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:02,542 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:16:02,556 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:02,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36730 deadline: 1732648622541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:02,557 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:02,557 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:02,557 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:16:02,660 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:02,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36770 deadline: 1732648622651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:02,661 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:02,661 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:02,661 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:16:02,673 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:02,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36730 deadline: 1732648622662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:02,673 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:02,673 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:02,674 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:16:02,882 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:02,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36770 deadline: 1732648622871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:02,882 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:02,882 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:02,882 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:16:02,883 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:02,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36730 deadline: 1732648622882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:02,884 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:02,884 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:02,884 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:16:02,919 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=376 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/C/2595a75695074136a78f01b230e4891d 2024-11-26T19:16:02,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/A/51215cb7cdf34728843fa1c4dd0a5574 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/A/51215cb7cdf34728843fa1c4dd0a5574 2024-11-26T19:16:02,927 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/A/51215cb7cdf34728843fa1c4dd0a5574, entries=150, sequenceid=376, filesize=12.0 K 2024-11-26T19:16:02,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/B/d09ffbdba8f1471783d06d0177318ad5 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/B/d09ffbdba8f1471783d06d0177318ad5 2024-11-26T19:16:02,931 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/B/d09ffbdba8f1471783d06d0177318ad5, entries=150, sequenceid=376, filesize=12.0 K 2024-11-26T19:16:02,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/C/2595a75695074136a78f01b230e4891d as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/C/2595a75695074136a78f01b230e4891d 2024-11-26T19:16:02,935 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/C/2595a75695074136a78f01b230e4891d, entries=150, sequenceid=376, filesize=12.0 K 2024-11-26T19:16:02,936 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 741bfc4b4e2b320db573e476b103383f in 869ms, sequenceid=376, compaction requested=false 2024-11-26T19:16:02,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for 741bfc4b4e2b320db573e476b103383f: 2024-11-26T19:16:02,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. 2024-11-26T19:16:02,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-11-26T19:16:02,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-11-26T19:16:02,939 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-11-26T19:16:02,939 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6320 sec 2024-11-26T19:16:02,941 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 1.6370 sec 2024-11-26T19:16:03,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(8855): Flush requested on 741bfc4b4e2b320db573e476b103383f 2024-11-26T19:16:03,197 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 741bfc4b4e2b320db573e476b103383f 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-26T19:16:03,198 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 741bfc4b4e2b320db573e476b103383f, store=A 2024-11-26T19:16:03,198 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:16:03,198 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 741bfc4b4e2b320db573e476b103383f, store=B 2024-11-26T19:16:03,198 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:16:03,198 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 741bfc4b4e2b320db573e476b103383f, store=C 2024-11-26T19:16:03,198 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:16:03,202 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/A/421fcf6402b94a2ca3c5a2dec03e15db is 50, key is test_row_0/A:col10/1732648563196/Put/seqid=0 2024-11-26T19:16:03,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742302_1478 (size=12301) 2024-11-26T19:16:03,207 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=391 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/A/421fcf6402b94a2ca3c5a2dec03e15db 2024-11-26T19:16:03,214 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/B/405151f3b7e64cce9610cf60729e30b8 is 50, key is test_row_0/B:col10/1732648563196/Put/seqid=0 2024-11-26T19:16:03,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742303_1479 (size=12301) 2024-11-26T19:16:03,280 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:03,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36770 deadline: 1732648623270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:03,280 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:03,280 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:03,280 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:16:03,285 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:03,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 207 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36730 deadline: 1732648623271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:03,285 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:03,285 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:03,285 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:16:03,402 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:03,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36770 deadline: 1732648623392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:03,402 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:03,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 208 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36730 deadline: 1732648623392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:03,402 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:03,402 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:03,402 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:16:03,402 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:03,403 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:03,403 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:16:03,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-26T19:16:03,452 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T19:16:03,452 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T19:16:03,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-11-26T19:16:03,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-26T19:16:03,454 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T19:16:03,455 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T19:16:03,455 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T19:16:03,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-26T19:16:03,607 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-26T19:16:03,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. 2024-11-26T19:16:03,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. as already flushing 2024-11-26T19:16:03,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. 2024-11-26T19:16:03,607 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:16:03,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:16:03,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:16:03,617 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:03,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36770 deadline: 1732648623612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:03,617 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:03,617 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:03,617 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:03,617 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:16:03,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 209 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36730 deadline: 1732648623612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:03,618 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:03,618 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:03,618 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:16:03,618 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=391 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/B/405151f3b7e64cce9610cf60729e30b8 2024-11-26T19:16:03,625 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/C/8cd22e96912c4ba49e35861c83070840 is 50, key is test_row_0/C:col10/1732648563196/Put/seqid=0 2024-11-26T19:16:03,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742304_1480 (size=12301) 2024-11-26T19:16:03,760 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-26T19:16:03,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. 2024-11-26T19:16:03,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. as already flushing 2024-11-26T19:16:03,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. 2024-11-26T19:16:03,760 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:16:03,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:16:03,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:16:03,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-26T19:16:03,913 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-26T19:16:03,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. 2024-11-26T19:16:03,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. as already flushing 2024-11-26T19:16:03,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. 2024-11-26T19:16:03,913 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:16:03,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:16:03,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:16:03,932 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:03,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36770 deadline: 1732648623922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:03,932 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:03,932 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:03,932 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:03,932 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:16:03,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 210 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36730 deadline: 1732648623922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:03,932 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:03,932 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:03,932 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:16:04,028 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=391 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/C/8cd22e96912c4ba49e35861c83070840 2024-11-26T19:16:04,032 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/A/421fcf6402b94a2ca3c5a2dec03e15db as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/A/421fcf6402b94a2ca3c5a2dec03e15db 2024-11-26T19:16:04,035 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/A/421fcf6402b94a2ca3c5a2dec03e15db, entries=150, sequenceid=391, filesize=12.0 K 2024-11-26T19:16:04,036 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/B/405151f3b7e64cce9610cf60729e30b8 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/B/405151f3b7e64cce9610cf60729e30b8 2024-11-26T19:16:04,039 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/B/405151f3b7e64cce9610cf60729e30b8, entries=150, sequenceid=391, filesize=12.0 K 2024-11-26T19:16:04,040 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/C/8cd22e96912c4ba49e35861c83070840 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/C/8cd22e96912c4ba49e35861c83070840 2024-11-26T19:16:04,043 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/C/8cd22e96912c4ba49e35861c83070840, entries=150, sequenceid=391, filesize=12.0 K 2024-11-26T19:16:04,043 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 741bfc4b4e2b320db573e476b103383f in 846ms, sequenceid=391, compaction requested=true 2024-11-26T19:16:04,043 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 741bfc4b4e2b320db573e476b103383f: 2024-11-26T19:16:04,044 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 741bfc4b4e2b320db573e476b103383f:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T19:16:04,044 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:16:04,044 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 741bfc4b4e2b320db573e476b103383f:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T19:16:04,044 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:16:04,044 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T19:16:04,044 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 741bfc4b4e2b320db573e476b103383f:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T19:16:04,044 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T19:16:04,044 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T19:16:04,044 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37381 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T19:16:04,044 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1541): 741bfc4b4e2b320db573e476b103383f/A is initiating minor compaction (all files) 2024-11-26T19:16:04,045 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 741bfc4b4e2b320db573e476b103383f/A in TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. 2024-11-26T19:16:04,045 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/A/d5569930f78f4ef4865c15d78bb28feb, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/A/51215cb7cdf34728843fa1c4dd0a5574, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/A/421fcf6402b94a2ca3c5a2dec03e15db] into tmpdir=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp, totalSize=36.5 K 2024-11-26T19:16:04,045 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37381 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T19:16:04,045 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1541): 741bfc4b4e2b320db573e476b103383f/B is initiating minor compaction (all files) 2024-11-26T19:16:04,045 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 741bfc4b4e2b320db573e476b103383f/B in TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. 2024-11-26T19:16:04,045 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/B/1705406432154a82a6d671962c9d71a9, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/B/d09ffbdba8f1471783d06d0177318ad5, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/B/405151f3b7e64cce9610cf60729e30b8] into tmpdir=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp, totalSize=36.5 K 2024-11-26T19:16:04,045 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting d5569930f78f4ef4865c15d78bb28feb, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1732648558276 2024-11-26T19:16:04,045 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting 1705406432154a82a6d671962c9d71a9, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1732648558276 2024-11-26T19:16:04,045 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting 51215cb7cdf34728843fa1c4dd0a5574, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=376, earliestPutTs=1732648561306 2024-11-26T19:16:04,045 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting d09ffbdba8f1471783d06d0177318ad5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=376, earliestPutTs=1732648561306 2024-11-26T19:16:04,046 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting 421fcf6402b94a2ca3c5a2dec03e15db, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=391, earliestPutTs=1732648562511 2024-11-26T19:16:04,046 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting 405151f3b7e64cce9610cf60729e30b8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=391, earliestPutTs=1732648562511 2024-11-26T19:16:04,052 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 741bfc4b4e2b320db573e476b103383f#A#compaction#407 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T19:16:04,052 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/A/86a9600b2d6840cca0b1597e726e67fe is 50, key is test_row_0/A:col10/1732648563196/Put/seqid=0 2024-11-26T19:16:04,052 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 741bfc4b4e2b320db573e476b103383f#B#compaction#408 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T19:16:04,053 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/B/e846ba6e090649c7a3b6d017d2d6262b is 50, key is test_row_0/B:col10/1732648563196/Put/seqid=0 2024-11-26T19:16:04,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742305_1481 (size=12881) 2024-11-26T19:16:04,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742306_1482 (size=12881) 2024-11-26T19:16:04,062 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/B/e846ba6e090649c7a3b6d017d2d6262b as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/B/e846ba6e090649c7a3b6d017d2d6262b 2024-11-26T19:16:04,065 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-26T19:16:04,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. 2024-11-26T19:16:04,066 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing 741bfc4b4e2b320db573e476b103383f 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-26T19:16:04,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 741bfc4b4e2b320db573e476b103383f, store=A 2024-11-26T19:16:04,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:16:04,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 741bfc4b4e2b320db573e476b103383f, store=B 2024-11-26T19:16:04,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:16:04,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 741bfc4b4e2b320db573e476b103383f, store=C 2024-11-26T19:16:04,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:16:04,069 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 741bfc4b4e2b320db573e476b103383f/B of 741bfc4b4e2b320db573e476b103383f into e846ba6e090649c7a3b6d017d2d6262b(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T19:16:04,069 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 741bfc4b4e2b320db573e476b103383f: 2024-11-26T19:16:04,069 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., storeName=741bfc4b4e2b320db573e476b103383f/B, priority=13, startTime=1732648564044; duration=0sec 2024-11-26T19:16:04,069 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T19:16:04,069 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 741bfc4b4e2b320db573e476b103383f:B 2024-11-26T19:16:04,069 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T19:16:04,070 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37381 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T19:16:04,070 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1541): 741bfc4b4e2b320db573e476b103383f/C is initiating minor compaction (all files) 2024-11-26T19:16:04,070 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 741bfc4b4e2b320db573e476b103383f/C in TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. 2024-11-26T19:16:04,070 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/C/49a99ab4394045bc857f9e20ac394ae6, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/C/2595a75695074136a78f01b230e4891d, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/C/8cd22e96912c4ba49e35861c83070840] into tmpdir=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp, totalSize=36.5 K 2024-11-26T19:16:04,070 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting 49a99ab4394045bc857f9e20ac394ae6, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1732648558276 2024-11-26T19:16:04,070 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting 2595a75695074136a78f01b230e4891d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=376, earliestPutTs=1732648561306 2024-11-26T19:16:04,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/A/b24a719dc418481bb804aeae68d4edd7 is 50, key is test_row_0/A:col10/1732648563239/Put/seqid=0 2024-11-26T19:16:04,071 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting 8cd22e96912c4ba49e35861c83070840, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=391, earliestPutTs=1732648562511 2024-11-26T19:16:04,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742307_1483 (size=12301) 2024-11-26T19:16:04,077 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 741bfc4b4e2b320db573e476b103383f#C#compaction#410 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T19:16:04,077 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/C/c7aee15270d0410fb440cc36174350f9 is 50, key is test_row_0/C:col10/1732648563196/Put/seqid=0 2024-11-26T19:16:04,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742308_1484 (size=12881) 2024-11-26T19:16:04,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-26T19:16:04,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(8855): Flush requested on 741bfc4b4e2b320db573e476b103383f 2024-11-26T19:16:04,449 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. as already flushing 2024-11-26T19:16:04,460 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/A/86a9600b2d6840cca0b1597e726e67fe as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/A/86a9600b2d6840cca0b1597e726e67fe 2024-11-26T19:16:04,469 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 741bfc4b4e2b320db573e476b103383f/A of 741bfc4b4e2b320db573e476b103383f into 86a9600b2d6840cca0b1597e726e67fe(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T19:16:04,469 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 741bfc4b4e2b320db573e476b103383f: 2024-11-26T19:16:04,469 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., storeName=741bfc4b4e2b320db573e476b103383f/A, priority=13, startTime=1732648564043; duration=0sec 2024-11-26T19:16:04,469 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:16:04,469 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 741bfc4b4e2b320db573e476b103383f:A 2024-11-26T19:16:04,476 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=411 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/A/b24a719dc418481bb804aeae68d4edd7 2024-11-26T19:16:04,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/B/6cbdc936a9f74a509a2f4af43bab250b is 50, key is test_row_0/B:col10/1732648563239/Put/seqid=0 2024-11-26T19:16:04,484 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/C/c7aee15270d0410fb440cc36174350f9 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/C/c7aee15270d0410fb440cc36174350f9 2024-11-26T19:16:04,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742309_1485 (size=12301) 2024-11-26T19:16:04,487 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=411 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/B/6cbdc936a9f74a509a2f4af43bab250b 2024-11-26T19:16:04,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/C/3337bedd73e542238f57cd4ceb1cf432 is 50, key is test_row_0/C:col10/1732648563239/Put/seqid=0 2024-11-26T19:16:04,493 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 741bfc4b4e2b320db573e476b103383f/C of 741bfc4b4e2b320db573e476b103383f into c7aee15270d0410fb440cc36174350f9(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T19:16:04,493 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 741bfc4b4e2b320db573e476b103383f: 2024-11-26T19:16:04,493 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., storeName=741bfc4b4e2b320db573e476b103383f/C, priority=13, startTime=1732648564044; duration=0sec 2024-11-26T19:16:04,493 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:16:04,493 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 741bfc4b4e2b320db573e476b103383f:C 2024-11-26T19:16:04,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742310_1486 (size=12301) 2024-11-26T19:16:04,505 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:04,505 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:04,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 207 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36770 deadline: 1732648624494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:04,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 216 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36730 deadline: 1732648624494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:04,506 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:04,506 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:04,506 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:04,506 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:04,506 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:16:04,506 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:16:04,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-26T19:16:04,619 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:04,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 208 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36770 deadline: 1732648624612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:04,619 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:04,619 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:04,619 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:04,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 217 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36730 deadline: 1732648624612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:04,619 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:16:04,620 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:04,620 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:04,620 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:16:04,828 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:04,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 209 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36770 deadline: 1732648624822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:04,829 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:04,829 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:04,829 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:04,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 218 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36730 deadline: 1732648624822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:04,829 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:16:04,829 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:04,829 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:04,829 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:16:04,898 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=411 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/C/3337bedd73e542238f57cd4ceb1cf432 2024-11-26T19:16:04,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/A/b24a719dc418481bb804aeae68d4edd7 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/A/b24a719dc418481bb804aeae68d4edd7 2024-11-26T19:16:04,906 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/A/b24a719dc418481bb804aeae68d4edd7, entries=150, sequenceid=411, filesize=12.0 K 2024-11-26T19:16:04,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/B/6cbdc936a9f74a509a2f4af43bab250b as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/B/6cbdc936a9f74a509a2f4af43bab250b 2024-11-26T19:16:04,912 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/B/6cbdc936a9f74a509a2f4af43bab250b, entries=150, sequenceid=411, filesize=12.0 K 2024-11-26T19:16:04,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/C/3337bedd73e542238f57cd4ceb1cf432 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/C/3337bedd73e542238f57cd4ceb1cf432 2024-11-26T19:16:04,916 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/C/3337bedd73e542238f57cd4ceb1cf432, entries=150, sequenceid=411, filesize=12.0 K 2024-11-26T19:16:04,917 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for 741bfc4b4e2b320db573e476b103383f in 851ms, sequenceid=411, compaction requested=false 2024-11-26T19:16:04,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for 741bfc4b4e2b320db573e476b103383f: 2024-11-26T19:16:04,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. 2024-11-26T19:16:04,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-11-26T19:16:04,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-11-26T19:16:04,922 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-11-26T19:16:04,922 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4640 sec 2024-11-26T19:16:04,924 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 1.4700 sec 2024-11-26T19:16:05,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(8855): Flush requested on 741bfc4b4e2b320db573e476b103383f 2024-11-26T19:16:05,137 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 741bfc4b4e2b320db573e476b103383f 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-26T19:16:05,137 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 741bfc4b4e2b320db573e476b103383f, store=A 2024-11-26T19:16:05,137 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:16:05,137 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 741bfc4b4e2b320db573e476b103383f, store=B 2024-11-26T19:16:05,137 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:16:05,137 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 741bfc4b4e2b320db573e476b103383f, store=C 2024-11-26T19:16:05,137 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:16:05,142 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/A/e0cf0d5c514b4888927e6f635569f746 is 50, key is test_row_0/A:col10/1732648565135/Put/seqid=0 2024-11-26T19:16:05,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742311_1487 (size=14741) 2024-11-26T19:16:05,147 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=428 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/A/e0cf0d5c514b4888927e6f635569f746 2024-11-26T19:16:05,154 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/B/6f1d7b4d17c047e8b9039227133d9f5e is 50, key is test_row_0/B:col10/1732648565135/Put/seqid=0 2024-11-26T19:16:05,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742312_1488 (size=12301) 2024-11-26T19:16:05,265 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:05,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 225 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36730 deadline: 1732648625249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:05,266 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:05,266 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:05,266 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:16:05,270 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:05,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 220 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36770 deadline: 1732648625266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:05,271 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:05,271 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:05,271 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:16:05,382 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:05,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 226 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36730 deadline: 1732648625372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:05,383 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:05,383 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:05,383 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:16:05,388 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:05,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 221 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36770 deadline: 1732648625381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:05,389 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:05,389 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:05,389 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:16:05,557 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=428 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/B/6f1d7b4d17c047e8b9039227133d9f5e 2024-11-26T19:16:05,564 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/C/99b7a53251cd4c35980090a1724febae is 50, key is test_row_0/C:col10/1732648565135/Put/seqid=0 2024-11-26T19:16:05,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742313_1489 (size=12301) 2024-11-26T19:16:05,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-26T19:16:05,601 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T19:16:05,602 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T19:16:05,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-11-26T19:16:05,603 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:05,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 227 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36730 deadline: 1732648625592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:05,603 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:05,603 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:05,603 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:16:05,604 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T19:16:05,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-26T19:16:05,604 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T19:16:05,604 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T19:16:05,607 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:05,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 222 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36770 deadline: 1732648625601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:05,608 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:05,608 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:05,608 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:16:05,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-26T19:16:05,756 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-26T19:16:05,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. 2024-11-26T19:16:05,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. as already flushing 2024-11-26T19:16:05,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. 2024-11-26T19:16:05,757 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:16:05,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:16:05,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:16:05,910 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-26T19:16:05,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. 2024-11-26T19:16:05,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. as already flushing 2024-11-26T19:16:05,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. 2024-11-26T19:16:05,910 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:16:05,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:16:05,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:16:05,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-26T19:16:05,923 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:05,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 228 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36730 deadline: 1732648625912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:05,923 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:05,924 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:05,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 223 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36770 deadline: 1732648625912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:05,924 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:05,924 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:16:05,924 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:05,924 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:05,924 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:16:05,968 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=428 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/C/99b7a53251cd4c35980090a1724febae 2024-11-26T19:16:05,972 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/A/e0cf0d5c514b4888927e6f635569f746 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/A/e0cf0d5c514b4888927e6f635569f746 2024-11-26T19:16:05,975 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/A/e0cf0d5c514b4888927e6f635569f746, entries=200, sequenceid=428, filesize=14.4 K 2024-11-26T19:16:05,975 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/B/6f1d7b4d17c047e8b9039227133d9f5e as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/B/6f1d7b4d17c047e8b9039227133d9f5e 2024-11-26T19:16:05,978 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/B/6f1d7b4d17c047e8b9039227133d9f5e, entries=150, sequenceid=428, filesize=12.0 K 2024-11-26T19:16:05,979 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/C/99b7a53251cd4c35980090a1724febae as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/C/99b7a53251cd4c35980090a1724febae 2024-11-26T19:16:05,982 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/C/99b7a53251cd4c35980090a1724febae, entries=150, sequenceid=428, filesize=12.0 K 2024-11-26T19:16:05,983 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 741bfc4b4e2b320db573e476b103383f in 846ms, sequenceid=428, compaction requested=true 2024-11-26T19:16:05,983 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 741bfc4b4e2b320db573e476b103383f: 2024-11-26T19:16:05,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 741bfc4b4e2b320db573e476b103383f:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T19:16:05,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:16:05,983 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T19:16:05,983 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T19:16:05,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 741bfc4b4e2b320db573e476b103383f:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T19:16:05,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:16:05,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 741bfc4b4e2b320db573e476b103383f:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T19:16:05,984 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T19:16:05,984 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39923 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T19:16:05,984 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1541): 741bfc4b4e2b320db573e476b103383f/A is initiating minor compaction (all files) 2024-11-26T19:16:05,984 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 741bfc4b4e2b320db573e476b103383f/A in TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. 2024-11-26T19:16:05,984 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/A/86a9600b2d6840cca0b1597e726e67fe, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/A/b24a719dc418481bb804aeae68d4edd7, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/A/e0cf0d5c514b4888927e6f635569f746] into tmpdir=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp, totalSize=39.0 K 2024-11-26T19:16:05,984 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T19:16:05,984 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1541): 741bfc4b4e2b320db573e476b103383f/B is initiating minor compaction (all files) 2024-11-26T19:16:05,984 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 741bfc4b4e2b320db573e476b103383f/B in TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. 2024-11-26T19:16:05,984 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting 86a9600b2d6840cca0b1597e726e67fe, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=391, earliestPutTs=1732648562511 2024-11-26T19:16:05,984 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/B/e846ba6e090649c7a3b6d017d2d6262b, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/B/6cbdc936a9f74a509a2f4af43bab250b, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/B/6f1d7b4d17c047e8b9039227133d9f5e] into tmpdir=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp, totalSize=36.6 K 2024-11-26T19:16:05,985 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting b24a719dc418481bb804aeae68d4edd7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=411, earliestPutTs=1732648563239 2024-11-26T19:16:05,985 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting e846ba6e090649c7a3b6d017d2d6262b, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=391, earliestPutTs=1732648562511 2024-11-26T19:16:05,985 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting e0cf0d5c514b4888927e6f635569f746, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=428, earliestPutTs=1732648564463 2024-11-26T19:16:05,985 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting 6cbdc936a9f74a509a2f4af43bab250b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=411, earliestPutTs=1732648563239 2024-11-26T19:16:05,985 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting 6f1d7b4d17c047e8b9039227133d9f5e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=428, earliestPutTs=1732648564486 2024-11-26T19:16:05,993 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 741bfc4b4e2b320db573e476b103383f#A#compaction#416 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T19:16:05,993 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/A/68646e8ca8b7464a9b5b9c0e0fde81d6 is 50, key is test_row_0/A:col10/1732648565135/Put/seqid=0 2024-11-26T19:16:05,997 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 741bfc4b4e2b320db573e476b103383f#B#compaction#417 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T19:16:05,997 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/B/55bb29036d684ae394d321aef9f248cb is 50, key is test_row_0/B:col10/1732648565135/Put/seqid=0 2024-11-26T19:16:06,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742314_1490 (size=12983) 2024-11-26T19:16:06,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742315_1491 (size=12983) 2024-11-26T19:16:06,063 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-26T19:16:06,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. 2024-11-26T19:16:06,068 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing 741bfc4b4e2b320db573e476b103383f 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-26T19:16:06,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 741bfc4b4e2b320db573e476b103383f, store=A 2024-11-26T19:16:06,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:16:06,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 741bfc4b4e2b320db573e476b103383f, store=B 2024-11-26T19:16:06,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:16:06,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 741bfc4b4e2b320db573e476b103383f, store=C 2024-11-26T19:16:06,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:16:06,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/A/1275d96c17604cbb89538bfffdb8d13f is 50, key is test_row_0/A:col10/1732648565221/Put/seqid=0 2024-11-26T19:16:06,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742316_1492 (size=12301) 2024-11-26T19:16:06,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-26T19:16:06,411 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/A/68646e8ca8b7464a9b5b9c0e0fde81d6 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/A/68646e8ca8b7464a9b5b9c0e0fde81d6 2024-11-26T19:16:06,415 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 741bfc4b4e2b320db573e476b103383f/A of 741bfc4b4e2b320db573e476b103383f into 68646e8ca8b7464a9b5b9c0e0fde81d6(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T19:16:06,415 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 741bfc4b4e2b320db573e476b103383f: 2024-11-26T19:16:06,415 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., storeName=741bfc4b4e2b320db573e476b103383f/A, priority=13, startTime=1732648565983; duration=0sec 2024-11-26T19:16:06,415 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T19:16:06,415 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 741bfc4b4e2b320db573e476b103383f:A 2024-11-26T19:16:06,416 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T19:16:06,417 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/B/55bb29036d684ae394d321aef9f248cb as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/B/55bb29036d684ae394d321aef9f248cb 2024-11-26T19:16:06,417 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T19:16:06,417 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1541): 741bfc4b4e2b320db573e476b103383f/C is initiating minor compaction (all files) 2024-11-26T19:16:06,417 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 741bfc4b4e2b320db573e476b103383f/C in TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. 2024-11-26T19:16:06,417 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/C/c7aee15270d0410fb440cc36174350f9, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/C/3337bedd73e542238f57cd4ceb1cf432, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/C/99b7a53251cd4c35980090a1724febae] into tmpdir=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp, totalSize=36.6 K 2024-11-26T19:16:06,417 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting c7aee15270d0410fb440cc36174350f9, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=391, earliestPutTs=1732648562511 2024-11-26T19:16:06,418 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3337bedd73e542238f57cd4ceb1cf432, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=411, earliestPutTs=1732648563239 2024-11-26T19:16:06,418 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting 99b7a53251cd4c35980090a1724febae, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=428, earliestPutTs=1732648564486 2024-11-26T19:16:06,421 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 741bfc4b4e2b320db573e476b103383f/B of 741bfc4b4e2b320db573e476b103383f into 55bb29036d684ae394d321aef9f248cb(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T19:16:06,421 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 741bfc4b4e2b320db573e476b103383f: 2024-11-26T19:16:06,421 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., storeName=741bfc4b4e2b320db573e476b103383f/B, priority=13, startTime=1732648565983; duration=0sec 2024-11-26T19:16:06,422 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:16:06,422 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 741bfc4b4e2b320db573e476b103383f:B 2024-11-26T19:16:06,425 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 741bfc4b4e2b320db573e476b103383f#C#compaction#419 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T19:16:06,426 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/C/d68665aac38e4843be68c6d7ebb3b976 is 50, key is test_row_0/C:col10/1732648565135/Put/seqid=0 2024-11-26T19:16:06,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742317_1493 (size=12983) 2024-11-26T19:16:06,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(8855): Flush requested on 741bfc4b4e2b320db573e476b103383f 2024-11-26T19:16:06,436 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. as already flushing 2024-11-26T19:16:06,476 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=446 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/A/1275d96c17604cbb89538bfffdb8d13f 2024-11-26T19:16:06,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/B/0e745d69292b4167aa3e14b40d38da5f is 50, key is test_row_0/B:col10/1732648565221/Put/seqid=0 2024-11-26T19:16:06,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742318_1494 (size=12301) 2024-11-26T19:16:06,490 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:06,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 229 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36770 deadline: 1732648626475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:06,490 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:06,490 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:06,490 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:16:06,504 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:06,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 235 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36730 deadline: 1732648626490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:06,505 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:06,505 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:06,505 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:16:06,612 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:06,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 230 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36770 deadline: 1732648626601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:06,613 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:06,613 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:06,613 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:16:06,615 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:06,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 236 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36730 deadline: 1732648626612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:06,615 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:06,615 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:06,615 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:16:06,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-26T19:16:06,830 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:06,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 231 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36770 deadline: 1732648626822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:06,830 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:06,831 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:06,831 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:06,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 237 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36730 deadline: 1732648626822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:06,831 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:16:06,832 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:06,832 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:06,832 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:16:06,834 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/C/d68665aac38e4843be68c6d7ebb3b976 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/C/d68665aac38e4843be68c6d7ebb3b976 2024-11-26T19:16:06,837 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 741bfc4b4e2b320db573e476b103383f/C of 741bfc4b4e2b320db573e476b103383f into d68665aac38e4843be68c6d7ebb3b976(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T19:16:06,837 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 741bfc4b4e2b320db573e476b103383f: 2024-11-26T19:16:06,837 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., storeName=741bfc4b4e2b320db573e476b103383f/C, priority=13, startTime=1732648565983; duration=0sec 2024-11-26T19:16:06,837 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:16:06,837 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 741bfc4b4e2b320db573e476b103383f:C 2024-11-26T19:16:06,887 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=446 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/B/0e745d69292b4167aa3e14b40d38da5f 2024-11-26T19:16:06,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/C/913e09399393429ebb6f09340c30c1cb is 50, key is test_row_0/C:col10/1732648565221/Put/seqid=0 2024-11-26T19:16:06,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742319_1495 (size=12301) 2024-11-26T19:16:07,142 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:07,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 232 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36770 deadline: 1732648627131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:07,143 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:07,143 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:07,143 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:16:07,150 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:07,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 238 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36730 deadline: 1732648627141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:07,150 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:07,150 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:07,150 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:16:07,298 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=446 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/C/913e09399393429ebb6f09340c30c1cb 2024-11-26T19:16:07,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/A/1275d96c17604cbb89538bfffdb8d13f as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/A/1275d96c17604cbb89538bfffdb8d13f 2024-11-26T19:16:07,305 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/A/1275d96c17604cbb89538bfffdb8d13f, entries=150, sequenceid=446, filesize=12.0 K 2024-11-26T19:16:07,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/B/0e745d69292b4167aa3e14b40d38da5f as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/B/0e745d69292b4167aa3e14b40d38da5f 2024-11-26T19:16:07,309 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/B/0e745d69292b4167aa3e14b40d38da5f, entries=150, sequenceid=446, filesize=12.0 K 2024-11-26T19:16:07,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/C/913e09399393429ebb6f09340c30c1cb as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/C/913e09399393429ebb6f09340c30c1cb 2024-11-26T19:16:07,313 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/C/913e09399393429ebb6f09340c30c1cb, entries=150, sequenceid=446, filesize=12.0 K 2024-11-26T19:16:07,314 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 741bfc4b4e2b320db573e476b103383f in 1251ms, sequenceid=446, compaction requested=false 2024-11-26T19:16:07,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for 741bfc4b4e2b320db573e476b103383f: 2024-11-26T19:16:07,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. 2024-11-26T19:16:07,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-11-26T19:16:07,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-11-26T19:16:07,317 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-11-26T19:16:07,317 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7110 sec 2024-11-26T19:16:07,319 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 1.7160 sec 2024-11-26T19:16:07,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(8855): Flush requested on 741bfc4b4e2b320db573e476b103383f 2024-11-26T19:16:07,659 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 741bfc4b4e2b320db573e476b103383f 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-26T19:16:07,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 741bfc4b4e2b320db573e476b103383f, store=A 2024-11-26T19:16:07,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:16:07,660 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 741bfc4b4e2b320db573e476b103383f, store=B 2024-11-26T19:16:07,660 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:16:07,660 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 741bfc4b4e2b320db573e476b103383f, store=C 2024-11-26T19:16:07,660 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:16:07,664 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/A/895ff6da1f74429ca4465d29c2017a8c is 50, key is test_row_0/A:col10/1732648567657/Put/seqid=0 2024-11-26T19:16:07,668 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742320_1496 (size=14741) 2024-11-26T19:16:07,727 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:07,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 241 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36770 deadline: 1732648627720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:07,727 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:07,727 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:07,727 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:16:07,734 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:07,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 246 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36730 deadline: 1732648627722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:07,735 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:07,735 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:07,735 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:16:07,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-26T19:16:07,752 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T19:16:07,752 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T19:16:07,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees 2024-11-26T19:16:07,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-26T19:16:07,754 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T19:16:07,754 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T19:16:07,754 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T19:16:07,770 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:07,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36758 deadline: 1732648627761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:07,770 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to fe76acf732b2,42785,1732648423970 for 'test_row_2' in 741bfc4b4e2b320db573e476b103383f of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18338 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T19:16:07,771 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:07,771 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:07,771 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:16:07,842 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:07,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 242 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36770 deadline: 1732648627832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:07,842 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:07,842 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:07,842 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:16:07,847 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:07,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 247 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36730 deadline: 1732648627841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:07,848 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:07,848 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:07,848 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:16:07,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-26T19:16:07,907 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-26T19:16:07,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. 2024-11-26T19:16:07,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. as already flushing 2024-11-26T19:16:07,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. 2024-11-26T19:16:07,907 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:16:07,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:16:07,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:16:08,060 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-26T19:16:08,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. 2024-11-26T19:16:08,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. as already flushing 2024-11-26T19:16:08,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. 2024-11-26T19:16:08,060 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:16:08,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:16:08,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:16:08,063 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:08,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 243 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36770 deadline: 1732648628052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:08,064 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:08,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 248 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36730 deadline: 1732648628052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:08,064 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:08,064 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:08,064 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:16:08,064 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:08,064 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:08,064 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:16:08,069 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=464 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/A/895ff6da1f74429ca4465d29c2017a8c 2024-11-26T19:16:08,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-26T19:16:08,075 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/B/3d1f91045b2f4e9dbfd73617cf8ee12c is 50, key is test_row_0/B:col10/1732648567657/Put/seqid=0 2024-11-26T19:16:08,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742321_1497 (size=12301) 2024-11-26T19:16:08,213 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-26T19:16:08,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. 2024-11-26T19:16:08,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. as already flushing 2024-11-26T19:16:08,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. 2024-11-26T19:16:08,213 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:16:08,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:16:08,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:16:08,366 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-26T19:16:08,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. 2024-11-26T19:16:08,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. as already flushing 2024-11-26T19:16:08,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. 2024-11-26T19:16:08,366 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:16:08,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:16:08,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:16:08,380 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:08,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 244 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36770 deadline: 1732648628372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:08,380 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:08,381 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:08,381 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:08,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 249 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36730 deadline: 1732648628372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:08,381 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:16:08,381 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:08,381 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:08,381 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:16:08,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-26T19:16:08,479 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=464 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/B/3d1f91045b2f4e9dbfd73617cf8ee12c 2024-11-26T19:16:08,485 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/C/36edecbc7e8742d7876f315d5896b46a is 50, key is test_row_0/C:col10/1732648567657/Put/seqid=0 2024-11-26T19:16:08,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742322_1498 (size=12301) 2024-11-26T19:16:08,519 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-26T19:16:08,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. 2024-11-26T19:16:08,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. as already flushing 2024-11-26T19:16:08,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. 2024-11-26T19:16:08,519 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:16:08,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:16:08,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:16:08,672 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-26T19:16:08,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. 2024-11-26T19:16:08,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. as already flushing 2024-11-26T19:16:08,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. 2024-11-26T19:16:08,672 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:16:08,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:16:08,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:16:08,825 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-26T19:16:08,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. 2024-11-26T19:16:08,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. as already flushing 2024-11-26T19:16:08,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. 2024-11-26T19:16:08,825 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:16:08,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:16:08,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:16:08,889 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=464 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/C/36edecbc7e8742d7876f315d5896b46a 2024-11-26T19:16:08,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-26T19:16:08,892 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/A/895ff6da1f74429ca4465d29c2017a8c as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/A/895ff6da1f74429ca4465d29c2017a8c 2024-11-26T19:16:08,896 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/A/895ff6da1f74429ca4465d29c2017a8c, entries=200, sequenceid=464, filesize=14.4 K 2024-11-26T19:16:08,896 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/B/3d1f91045b2f4e9dbfd73617cf8ee12c as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/B/3d1f91045b2f4e9dbfd73617cf8ee12c 2024-11-26T19:16:08,899 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/B/3d1f91045b2f4e9dbfd73617cf8ee12c, entries=150, sequenceid=464, filesize=12.0 K 2024-11-26T19:16:08,900 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/C/36edecbc7e8742d7876f315d5896b46a as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/C/36edecbc7e8742d7876f315d5896b46a 2024-11-26T19:16:08,902 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/C/36edecbc7e8742d7876f315d5896b46a, entries=150, sequenceid=464, filesize=12.0 K 2024-11-26T19:16:08,903 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 741bfc4b4e2b320db573e476b103383f in 1244ms, sequenceid=464, compaction requested=true 2024-11-26T19:16:08,903 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 741bfc4b4e2b320db573e476b103383f: 2024-11-26T19:16:08,904 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 741bfc4b4e2b320db573e476b103383f:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T19:16:08,904 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T19:16:08,904 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:16:08,904 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 741bfc4b4e2b320db573e476b103383f:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T19:16:08,904 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:16:08,904 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 741bfc4b4e2b320db573e476b103383f:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T19:16:08,904 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T19:16:08,904 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T19:16:08,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(8855): Flush requested on 741bfc4b4e2b320db573e476b103383f 2024-11-26T19:16:08,904 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 741bfc4b4e2b320db573e476b103383f 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-26T19:16:08,904 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 741bfc4b4e2b320db573e476b103383f, store=A 2024-11-26T19:16:08,904 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:16:08,904 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 741bfc4b4e2b320db573e476b103383f, store=B 2024-11-26T19:16:08,904 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:16:08,905 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 741bfc4b4e2b320db573e476b103383f, store=C 2024-11-26T19:16:08,905 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:16:08,905 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T19:16:08,905 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40025 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T19:16:08,905 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1541): 741bfc4b4e2b320db573e476b103383f/B is initiating minor compaction (all files) 2024-11-26T19:16:08,905 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1541): 741bfc4b4e2b320db573e476b103383f/A is initiating minor compaction (all files) 2024-11-26T19:16:08,905 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 741bfc4b4e2b320db573e476b103383f/B in TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. 2024-11-26T19:16:08,905 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 741bfc4b4e2b320db573e476b103383f/A in TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. 2024-11-26T19:16:08,905 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/B/55bb29036d684ae394d321aef9f248cb, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/B/0e745d69292b4167aa3e14b40d38da5f, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/B/3d1f91045b2f4e9dbfd73617cf8ee12c] into tmpdir=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp, totalSize=36.7 K 2024-11-26T19:16:08,905 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/A/68646e8ca8b7464a9b5b9c0e0fde81d6, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/A/1275d96c17604cbb89538bfffdb8d13f, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/A/895ff6da1f74429ca4465d29c2017a8c] into tmpdir=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp, totalSize=39.1 K 2024-11-26T19:16:08,906 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting 55bb29036d684ae394d321aef9f248cb, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=428, earliestPutTs=1732648564486 2024-11-26T19:16:08,906 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting 68646e8ca8b7464a9b5b9c0e0fde81d6, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=428, earliestPutTs=1732648564486 2024-11-26T19:16:08,906 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting 0e745d69292b4167aa3e14b40d38da5f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=446, earliestPutTs=1732648565219 2024-11-26T19:16:08,907 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1275d96c17604cbb89538bfffdb8d13f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=446, earliestPutTs=1732648565219 2024-11-26T19:16:08,907 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting 3d1f91045b2f4e9dbfd73617cf8ee12c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=464, earliestPutTs=1732648566460 2024-11-26T19:16:08,907 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting 895ff6da1f74429ca4465d29c2017a8c, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=464, earliestPutTs=1732648566460 2024-11-26T19:16:08,914 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/A/17950f70b3a345168667fc1bbb56b274 is 50, key is test_row_0/A:col10/1732648567686/Put/seqid=0 2024-11-26T19:16:08,921 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 741bfc4b4e2b320db573e476b103383f#B#compaction#426 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T19:16:08,922 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/B/40ebdb8eee39435eab3336deb50ddfd3 is 50, key is test_row_0/B:col10/1732648567657/Put/seqid=0 2024-11-26T19:16:08,927 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 741bfc4b4e2b320db573e476b103383f#A#compaction#427 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T19:16:08,927 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/A/5b2ba53ee2d24b6da630a570c88e1573 is 50, key is test_row_0/A:col10/1732648567657/Put/seqid=0 2024-11-26T19:16:08,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742323_1499 (size=14741) 2024-11-26T19:16:08,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742324_1500 (size=13085) 2024-11-26T19:16:08,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742325_1501 (size=13085) 2024-11-26T19:16:08,948 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/B/40ebdb8eee39435eab3336deb50ddfd3 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/B/40ebdb8eee39435eab3336deb50ddfd3 2024-11-26T19:16:08,952 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 741bfc4b4e2b320db573e476b103383f/B of 741bfc4b4e2b320db573e476b103383f into 40ebdb8eee39435eab3336deb50ddfd3(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T19:16:08,952 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 741bfc4b4e2b320db573e476b103383f: 2024-11-26T19:16:08,952 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., storeName=741bfc4b4e2b320db573e476b103383f/B, priority=13, startTime=1732648568904; duration=0sec 2024-11-26T19:16:08,952 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T19:16:08,952 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 741bfc4b4e2b320db573e476b103383f:B 2024-11-26T19:16:08,952 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T19:16:08,953 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T19:16:08,953 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1541): 741bfc4b4e2b320db573e476b103383f/C is initiating minor compaction (all files) 2024-11-26T19:16:08,953 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 741bfc4b4e2b320db573e476b103383f/C in TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. 2024-11-26T19:16:08,953 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/C/d68665aac38e4843be68c6d7ebb3b976, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/C/913e09399393429ebb6f09340c30c1cb, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/C/36edecbc7e8742d7876f315d5896b46a] into tmpdir=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp, totalSize=36.7 K 2024-11-26T19:16:08,954 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting d68665aac38e4843be68c6d7ebb3b976, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=428, earliestPutTs=1732648564486 2024-11-26T19:16:08,954 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting 913e09399393429ebb6f09340c30c1cb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=446, earliestPutTs=1732648565219 2024-11-26T19:16:08,954 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting 36edecbc7e8742d7876f315d5896b46a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=464, earliestPutTs=1732648566460 2024-11-26T19:16:08,960 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 741bfc4b4e2b320db573e476b103383f#C#compaction#428 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T19:16:08,960 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/C/b0720d744826422182045cb6f81fab79 is 50, key is test_row_0/C:col10/1732648567657/Put/seqid=0 2024-11-26T19:16:08,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742326_1502 (size=13085) 2024-11-26T19:16:08,971 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:08,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 251 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36770 deadline: 1732648628963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:08,971 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:08,971 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:08,971 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:16:08,978 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-26T19:16:08,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. 2024-11-26T19:16:08,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. as already flushing 2024-11-26T19:16:08,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. 2024-11-26T19:16:08,978 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:16:08,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:16:08,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:16:08,980 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:08,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 256 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36730 deadline: 1732648628965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:08,981 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:08,981 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:08,981 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:16:09,089 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:09,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 252 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36770 deadline: 1732648629081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:09,089 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:09,089 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:09,089 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:16:09,101 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:09,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 257 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36730 deadline: 1732648629092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:09,101 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:09,101 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:09,101 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:16:09,130 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-26T19:16:09,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. 2024-11-26T19:16:09,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. as already flushing 2024-11-26T19:16:09,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. 2024-11-26T19:16:09,131 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:16:09,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:16:09,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:16:09,284 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-26T19:16:09,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. 2024-11-26T19:16:09,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. as already flushing 2024-11-26T19:16:09,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. 2024-11-26T19:16:09,284 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:16:09,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:16:09,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:16:09,297 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:09,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 253 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36770 deadline: 1732648629292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:09,297 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:09,297 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:09,297 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:16:09,321 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:09,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 258 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36730 deadline: 1732648629312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:09,322 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:09,322 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:09,322 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:16:09,342 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=482 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/A/17950f70b3a345168667fc1bbb56b274 2024-11-26T19:16:09,342 INFO [Thread-1951 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1951. 2024-11-26T19:16:09,342 DEBUG [Thread-1951 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T19:16:09,342 DEBUG [Thread-1951 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:16:09,342 DEBUG [Thread-1951 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:16:09,342 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T19:16:09,350 INFO [Thread-1949 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1949. 2024-11-26T19:16:09,350 DEBUG [Thread-1949 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T19:16:09,350 DEBUG [Thread-1949 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:16:09,350 DEBUG [Thread-1949 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:16:09,350 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T19:16:09,350 INFO [Thread-1953 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1953. 2024-11-26T19:16:09,350 DEBUG [Thread-1953 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T19:16:09,351 DEBUG [Thread-1953 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:16:09,351 DEBUG [Thread-1953 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:16:09,351 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T19:16:09,351 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/A/5b2ba53ee2d24b6da630a570c88e1573 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/A/5b2ba53ee2d24b6da630a570c88e1573 2024-11-26T19:16:09,355 INFO [Thread-1945 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1945. 2024-11-26T19:16:09,355 DEBUG [Thread-1945 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T19:16:09,355 DEBUG [Thread-1945 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:16:09,355 DEBUG [Thread-1945 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:16:09,355 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/B/04c777ab07e0436db08fde67d2420c8c is 50, key is test_row_0/B:col10/1732648567686/Put/seqid=0 2024-11-26T19:16:09,355 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T19:16:09,356 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 741bfc4b4e2b320db573e476b103383f/A of 741bfc4b4e2b320db573e476b103383f into 5b2ba53ee2d24b6da630a570c88e1573(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T19:16:09,356 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 741bfc4b4e2b320db573e476b103383f: 2024-11-26T19:16:09,356 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., storeName=741bfc4b4e2b320db573e476b103383f/A, priority=13, startTime=1732648568903; duration=0sec 2024-11-26T19:16:09,356 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:16:09,356 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 741bfc4b4e2b320db573e476b103383f:A 2024-11-26T19:16:09,359 INFO [Thread-1947 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1947. 2024-11-26T19:16:09,359 DEBUG [Thread-1947 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T19:16:09,359 DEBUG [Thread-1947 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:16:09,359 DEBUG [Thread-1947 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:16:09,359 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T19:16:09,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742327_1503 (size=12301) 2024-11-26T19:16:09,373 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/C/b0720d744826422182045cb6f81fab79 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/C/b0720d744826422182045cb6f81fab79 2024-11-26T19:16:09,376 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 741bfc4b4e2b320db573e476b103383f/C of 741bfc4b4e2b320db573e476b103383f into b0720d744826422182045cb6f81fab79(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T19:16:09,376 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 741bfc4b4e2b320db573e476b103383f: 2024-11-26T19:16:09,376 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., storeName=741bfc4b4e2b320db573e476b103383f/C, priority=13, startTime=1732648568904; duration=0sec 2024-11-26T19:16:09,376 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:16:09,376 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 741bfc4b4e2b320db573e476b103383f:C 2024-11-26T19:16:09,436 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-26T19:16:09,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. 2024-11-26T19:16:09,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. as already flushing 2024-11-26T19:16:09,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. 2024-11-26T19:16:09,437 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:16:09,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:16:09,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:16:09,590 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-26T19:16:09,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. 2024-11-26T19:16:09,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. as already flushing 2024-11-26T19:16:09,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. 2024-11-26T19:16:09,590 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:16:09,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:16:09,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:16:09,602 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:09,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 254 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36770 deadline: 1732648629602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:09,603 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:09,603 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:09,603 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:16:09,632 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:09,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 259 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36730 deadline: 1732648629631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:09,632 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:09,632 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:09,632 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:16:09,743 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-26T19:16:09,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. 2024-11-26T19:16:09,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. as already flushing 2024-11-26T19:16:09,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. 2024-11-26T19:16:09,743 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:16:09,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:16:09,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:16:09,764 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=482 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/B/04c777ab07e0436db08fde67d2420c8c 2024-11-26T19:16:09,769 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/C/13d1f43522664de7be30b89dd2053eb2 is 50, key is test_row_0/C:col10/1732648567686/Put/seqid=0 2024-11-26T19:16:09,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742328_1504 (size=12301) 2024-11-26T19:16:09,896 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-26T19:16:09,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. 2024-11-26T19:16:09,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. as already flushing 2024-11-26T19:16:09,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. 2024-11-26T19:16:09,896 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:16:09,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:16:09,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:16:09,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-26T19:16:09,932 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:09,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36712 deadline: 1732648629932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:09,932 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to fe76acf732b2,42785,1732648423970 for 'test_row_2' in 741bfc4b4e2b320db573e476b103383f of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18292 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T19:16:09,933 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:09,933 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:09,933 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:16:09,972 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:09,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36744 deadline: 1732648629972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:09,973 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to fe76acf732b2,42785,1732648423970 for 'test_row_2' in 741bfc4b4e2b320db573e476b103383f of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18340 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T19:16:09,973 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:09,973 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:09,973 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:16:10,049 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-26T19:16:10,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. 2024-11-26T19:16:10,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. as already flushing 2024-11-26T19:16:10,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. 2024-11-26T19:16:10,049 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:16:10,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:16:10,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:16:10,112 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:10,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 255 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36770 deadline: 1732648630112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:10,113 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:10,113 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:10,113 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:16:10,142 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:10,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 260 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36730 deadline: 1732648630141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:10,142 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 , the old value is region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:10,142 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:10,142 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., hostname=fe76acf732b2,42785,1732648423970, seqNum=2 because the exception is null or not the one we care about 2024-11-26T19:16:10,173 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=482 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/C/13d1f43522664de7be30b89dd2053eb2 2024-11-26T19:16:10,176 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/A/17950f70b3a345168667fc1bbb56b274 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/A/17950f70b3a345168667fc1bbb56b274 2024-11-26T19:16:10,179 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/A/17950f70b3a345168667fc1bbb56b274, entries=200, sequenceid=482, filesize=14.4 K 2024-11-26T19:16:10,179 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/B/04c777ab07e0436db08fde67d2420c8c as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/B/04c777ab07e0436db08fde67d2420c8c 2024-11-26T19:16:10,182 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/B/04c777ab07e0436db08fde67d2420c8c, entries=150, sequenceid=482, filesize=12.0 K 2024-11-26T19:16:10,182 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/C/13d1f43522664de7be30b89dd2053eb2 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/C/13d1f43522664de7be30b89dd2053eb2 2024-11-26T19:16:10,184 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/C/13d1f43522664de7be30b89dd2053eb2, entries=150, sequenceid=482, filesize=12.0 K 2024-11-26T19:16:10,185 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 741bfc4b4e2b320db573e476b103383f in 1281ms, sequenceid=482, compaction requested=false 2024-11-26T19:16:10,185 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 741bfc4b4e2b320db573e476b103383f: 2024-11-26T19:16:10,201 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-26T19:16:10,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. 2024-11-26T19:16:10,202 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2902): Flushing 741bfc4b4e2b320db573e476b103383f 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-26T19:16:10,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 741bfc4b4e2b320db573e476b103383f, store=A 2024-11-26T19:16:10,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:16:10,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 741bfc4b4e2b320db573e476b103383f, store=B 2024-11-26T19:16:10,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:16:10,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 741bfc4b4e2b320db573e476b103383f, store=C 2024-11-26T19:16:10,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:16:10,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/A/baa49d8ee9cc442083ab97d211abbb62 is 50, key is test_row_0/A:col10/1732648568942/Put/seqid=0 2024-11-26T19:16:10,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742329_1505 (size=12301) 2024-11-26T19:16:10,608 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=499 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/A/baa49d8ee9cc442083ab97d211abbb62 2024-11-26T19:16:10,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/B/2bdab127d4694b859d037e9759a41353 is 50, key is test_row_0/B:col10/1732648568942/Put/seqid=0 2024-11-26T19:16:10,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742330_1506 (size=12301) 2024-11-26T19:16:11,018 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=499 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/B/2bdab127d4694b859d037e9759a41353 2024-11-26T19:16:11,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/C/7f30c26386fa402c9d9066bd9a79ce46 is 50, key is test_row_0/C:col10/1732648568942/Put/seqid=0 2024-11-26T19:16:11,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742331_1507 (size=12301) 2024-11-26T19:16:11,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(8855): Flush requested on 741bfc4b4e2b320db573e476b103383f 2024-11-26T19:16:11,134 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. as already flushing 2024-11-26T19:16:11,134 INFO [Thread-1940 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1940. 2024-11-26T19:16:11,134 DEBUG [Thread-1940 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T19:16:11,134 DEBUG [Thread-1940 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:16:11,134 DEBUG [Thread-1940 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:16:11,134 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T19:16:11,154 INFO [Thread-1936 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1936. 2024-11-26T19:16:11,154 DEBUG [Thread-1936 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T19:16:11,154 DEBUG [Thread-1936 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:16:11,154 DEBUG [Thread-1936 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:16:11,155 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T19:16:11,431 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=499 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/C/7f30c26386fa402c9d9066bd9a79ce46 2024-11-26T19:16:11,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/A/baa49d8ee9cc442083ab97d211abbb62 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/A/baa49d8ee9cc442083ab97d211abbb62 2024-11-26T19:16:11,437 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/A/baa49d8ee9cc442083ab97d211abbb62, entries=150, sequenceid=499, filesize=12.0 K 2024-11-26T19:16:11,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/B/2bdab127d4694b859d037e9759a41353 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/B/2bdab127d4694b859d037e9759a41353 2024-11-26T19:16:11,440 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/B/2bdab127d4694b859d037e9759a41353, entries=150, sequenceid=499, filesize=12.0 K 2024-11-26T19:16:11,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/C/7f30c26386fa402c9d9066bd9a79ce46 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/C/7f30c26386fa402c9d9066bd9a79ce46 2024-11-26T19:16:11,443 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/C/7f30c26386fa402c9d9066bd9a79ce46, entries=150, sequenceid=499, filesize=12.0 K 2024-11-26T19:16:11,444 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=13.42 KB/13740 for 741bfc4b4e2b320db573e476b103383f in 1243ms, sequenceid=499, compaction requested=true 2024-11-26T19:16:11,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2603): Flush status journal for 741bfc4b4e2b320db573e476b103383f: 2024-11-26T19:16:11,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. 2024-11-26T19:16:11,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132 2024-11-26T19:16:11,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster(4169): Remote procedure done, pid=132 2024-11-26T19:16:11,446 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-11-26T19:16:11,446 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.6910 sec 2024-11-26T19:16:11,448 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees in 3.6950 sec 2024-11-26T19:16:11,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-26T19:16:11,922 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T19:16:12,135 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-26T19:16:17,873 INFO [Thread-1938 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1938. 2024-11-26T19:16:17,873 DEBUG [Thread-1938 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T19:16:17,873 DEBUG [Thread-1938 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:16:17,873 DEBUG [Thread-1938 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:16:17,873 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T19:16:19,974 INFO [Thread-1942 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1942. 2024-11-26T19:16:19,974 DEBUG [Thread-1942 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T19:16:19,974 DEBUG [Thread-1942 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:16:19,974 DEBUG [Thread-1942 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:16:19,974 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T19:16:20,063 INFO [Thread-1934 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1934. 2024-11-26T19:16:20,063 DEBUG [Thread-1934 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T19:16:20,063 DEBUG [Thread-1934 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:16:20,063 DEBUG [Thread-1934 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:16:20,063 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T19:16:20,064 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-26T19:16:20,064 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 10 2024-11-26T19:16:20,064 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 200 2024-11-26T19:16:20,064 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 9 2024-11-26T19:16:20,064 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 195 2024-11-26T19:16:20,064 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 10 2024-11-26T19:16:20,064 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-26T19:16:20,064 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-26T19:16:20,064 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1125 2024-11-26T19:16:20,064 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3372 rows 2024-11-26T19:16:20,064 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1127 2024-11-26T19:16:20,064 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3376 rows 2024-11-26T19:16:20,064 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1122 2024-11-26T19:16:20,064 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3365 rows 2024-11-26T19:16:20,064 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1125 2024-11-26T19:16:20,064 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3374 rows 2024-11-26T19:16:20,064 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1123 2024-11-26T19:16:20,064 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3368 rows 2024-11-26T19:16:20,064 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-26T19:16:20,064 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-26T19:16:20,064 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:16:20,064 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:16:20,064 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T19:16:20,065 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-26T19:16:20,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-26T19:16:20,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-26T19:16:20,067 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732648580067"}]},"ts":"1732648580067"} 2024-11-26T19:16:20,069 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-26T19:16:20,069 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-26T19:16:20,069 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-26T19:16:20,070 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=135, ppid=134, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=741bfc4b4e2b320db573e476b103383f, UNASSIGN}] 2024-11-26T19:16:20,071 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=135, ppid=134, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=741bfc4b4e2b320db573e476b103383f, UNASSIGN 2024-11-26T19:16:20,072 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=135 updating hbase:meta row=741bfc4b4e2b320db573e476b103383f, regionState=CLOSING, regionLocation=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:20,073 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=135, ppid=134, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=741bfc4b4e2b320db573e476b103383f, UNASSIGN because future has completed 2024-11-26T19:16:20,073 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-26T19:16:20,073 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE, hasLock=false; CloseRegionProcedure 741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970}] 2024-11-26T19:16:20,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-26T19:16:20,225 INFO [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] handler.UnassignRegionHandler(122): Close 741bfc4b4e2b320db573e476b103383f 2024-11-26T19:16:20,225 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-26T19:16:20,225 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1722): Closing 741bfc4b4e2b320db573e476b103383f, disabling compactions & flushes 2024-11-26T19:16:20,225 INFO [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. 2024-11-26T19:16:20,225 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. 2024-11-26T19:16:20,225 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. after waiting 0 ms 2024-11-26T19:16:20,225 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. 2024-11-26T19:16:20,225 INFO [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(2902): Flushing 741bfc4b4e2b320db573e476b103383f 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-26T19:16:20,226 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 741bfc4b4e2b320db573e476b103383f, store=A 2024-11-26T19:16:20,226 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:16:20,226 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 741bfc4b4e2b320db573e476b103383f, store=B 2024-11-26T19:16:20,226 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:16:20,226 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 741bfc4b4e2b320db573e476b103383f, store=C 2024-11-26T19:16:20,226 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:16:20,229 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/A/d63caad32301424fa4644cf18bca1f2c is 50, key is test_row_0/A:col10/1732648571153/Put/seqid=0 2024-11-26T19:16:20,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742332_1508 (size=9857) 2024-11-26T19:16:20,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-26T19:16:20,632 INFO [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=507 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/A/d63caad32301424fa4644cf18bca1f2c 2024-11-26T19:16:20,638 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/B/bdaf4b1b8d4a473e8d4fe56d99657983 is 50, key is test_row_0/B:col10/1732648571153/Put/seqid=0 2024-11-26T19:16:20,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742333_1509 (size=9857) 2024-11-26T19:16:20,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-26T19:16:21,041 INFO [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=507 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/B/bdaf4b1b8d4a473e8d4fe56d99657983 2024-11-26T19:16:21,046 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/C/5baae9167df3467f82ab0da7f8a7ffa8 is 50, key is test_row_0/C:col10/1732648571153/Put/seqid=0 2024-11-26T19:16:21,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742334_1510 (size=9857) 2024-11-26T19:16:21,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-26T19:16:21,450 INFO [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=507 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/C/5baae9167df3467f82ab0da7f8a7ffa8 2024-11-26T19:16:21,453 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/A/d63caad32301424fa4644cf18bca1f2c as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/A/d63caad32301424fa4644cf18bca1f2c 2024-11-26T19:16:21,456 INFO [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/A/d63caad32301424fa4644cf18bca1f2c, entries=100, sequenceid=507, filesize=9.6 K 2024-11-26T19:16:21,456 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/B/bdaf4b1b8d4a473e8d4fe56d99657983 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/B/bdaf4b1b8d4a473e8d4fe56d99657983 2024-11-26T19:16:21,459 INFO [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/B/bdaf4b1b8d4a473e8d4fe56d99657983, entries=100, sequenceid=507, filesize=9.6 K 2024-11-26T19:16:21,459 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/.tmp/C/5baae9167df3467f82ab0da7f8a7ffa8 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/C/5baae9167df3467f82ab0da7f8a7ffa8 2024-11-26T19:16:21,462 INFO [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/C/5baae9167df3467f82ab0da7f8a7ffa8, entries=100, sequenceid=507, filesize=9.6 K 2024-11-26T19:16:21,462 INFO [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 741bfc4b4e2b320db573e476b103383f in 1237ms, sequenceid=507, compaction requested=true 2024-11-26T19:16:21,463 DEBUG [StoreCloser-TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/A/cc13134c158940ce9742b4f24755e256, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/A/f55c1a2a88a14a7dbd27c6dc4a6f6498, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/A/23a12e07036e4c2985e6151fc47168b6, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/A/2c11fcb559cf4f5da02cfe3ffd64953b, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/A/b3b54c3af4e7486785d2c69746501409, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/A/7d05fe7933c3443eb046fbe7f56a2bd6, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/A/b51590d8309f467aa918f37bd8bf1174, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/A/57ef3666ef6842c7871c2a65c3237ba0, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/A/d3c4f2ba6b854fcdadcf12fdac421d6b, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/A/5ce2cea0e1c547f983371f4f19c452a6, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/A/4628ad5741c74632957a7a75dae34a05, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/A/505bd621263f44f7895e792df03316ce, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/A/da0d7b2a5baf47af93c8ec99a4df5dbe, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/A/3deed8cf987f46ee94d52b29f05ffd1f, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/A/d5569930f78f4ef4865c15d78bb28feb, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/A/51215cb7cdf34728843fa1c4dd0a5574, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/A/86a9600b2d6840cca0b1597e726e67fe, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/A/421fcf6402b94a2ca3c5a2dec03e15db, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/A/b24a719dc418481bb804aeae68d4edd7, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/A/e0cf0d5c514b4888927e6f635569f746, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/A/68646e8ca8b7464a9b5b9c0e0fde81d6, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/A/1275d96c17604cbb89538bfffdb8d13f, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/A/895ff6da1f74429ca4465d29c2017a8c] to archive 2024-11-26T19:16:21,464 DEBUG [StoreCloser-TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-26T19:16:21,465 DEBUG [StoreCloser-TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/A/cc13134c158940ce9742b4f24755e256 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/A/cc13134c158940ce9742b4f24755e256 2024-11-26T19:16:21,466 DEBUG [StoreCloser-TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/A/f55c1a2a88a14a7dbd27c6dc4a6f6498 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/A/f55c1a2a88a14a7dbd27c6dc4a6f6498 2024-11-26T19:16:21,467 DEBUG [StoreCloser-TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/A/23a12e07036e4c2985e6151fc47168b6 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/A/23a12e07036e4c2985e6151fc47168b6 2024-11-26T19:16:21,467 DEBUG [StoreCloser-TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/A/2c11fcb559cf4f5da02cfe3ffd64953b to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/A/2c11fcb559cf4f5da02cfe3ffd64953b 2024-11-26T19:16:21,468 DEBUG [StoreCloser-TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/A/b3b54c3af4e7486785d2c69746501409 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/A/b3b54c3af4e7486785d2c69746501409 2024-11-26T19:16:21,469 DEBUG [StoreCloser-TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/A/7d05fe7933c3443eb046fbe7f56a2bd6 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/A/7d05fe7933c3443eb046fbe7f56a2bd6 2024-11-26T19:16:21,470 DEBUG [StoreCloser-TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/A/b51590d8309f467aa918f37bd8bf1174 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/A/b51590d8309f467aa918f37bd8bf1174 2024-11-26T19:16:21,470 DEBUG [StoreCloser-TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/A/57ef3666ef6842c7871c2a65c3237ba0 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/A/57ef3666ef6842c7871c2a65c3237ba0 2024-11-26T19:16:21,471 DEBUG [StoreCloser-TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/A/d3c4f2ba6b854fcdadcf12fdac421d6b to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/A/d3c4f2ba6b854fcdadcf12fdac421d6b 2024-11-26T19:16:21,472 DEBUG [StoreCloser-TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/A/5ce2cea0e1c547f983371f4f19c452a6 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/A/5ce2cea0e1c547f983371f4f19c452a6 2024-11-26T19:16:21,472 DEBUG [StoreCloser-TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/A/4628ad5741c74632957a7a75dae34a05 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/A/4628ad5741c74632957a7a75dae34a05 2024-11-26T19:16:21,473 DEBUG [StoreCloser-TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/A/505bd621263f44f7895e792df03316ce to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/A/505bd621263f44f7895e792df03316ce 2024-11-26T19:16:21,474 DEBUG [StoreCloser-TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/A/da0d7b2a5baf47af93c8ec99a4df5dbe to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/A/da0d7b2a5baf47af93c8ec99a4df5dbe 2024-11-26T19:16:21,474 DEBUG [StoreCloser-TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/A/3deed8cf987f46ee94d52b29f05ffd1f to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/A/3deed8cf987f46ee94d52b29f05ffd1f 2024-11-26T19:16:21,475 DEBUG [StoreCloser-TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/A/d5569930f78f4ef4865c15d78bb28feb to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/A/d5569930f78f4ef4865c15d78bb28feb 2024-11-26T19:16:21,476 DEBUG [StoreCloser-TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/A/51215cb7cdf34728843fa1c4dd0a5574 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/A/51215cb7cdf34728843fa1c4dd0a5574 2024-11-26T19:16:21,476 DEBUG [StoreCloser-TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/A/86a9600b2d6840cca0b1597e726e67fe to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/A/86a9600b2d6840cca0b1597e726e67fe 2024-11-26T19:16:21,477 DEBUG [StoreCloser-TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/A/421fcf6402b94a2ca3c5a2dec03e15db to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/A/421fcf6402b94a2ca3c5a2dec03e15db 2024-11-26T19:16:21,478 DEBUG [StoreCloser-TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/A/b24a719dc418481bb804aeae68d4edd7 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/A/b24a719dc418481bb804aeae68d4edd7 2024-11-26T19:16:21,479 DEBUG [StoreCloser-TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/A/e0cf0d5c514b4888927e6f635569f746 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/A/e0cf0d5c514b4888927e6f635569f746 2024-11-26T19:16:21,479 DEBUG [StoreCloser-TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/A/68646e8ca8b7464a9b5b9c0e0fde81d6 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/A/68646e8ca8b7464a9b5b9c0e0fde81d6 2024-11-26T19:16:21,480 DEBUG [StoreCloser-TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/A/1275d96c17604cbb89538bfffdb8d13f to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/A/1275d96c17604cbb89538bfffdb8d13f 2024-11-26T19:16:21,481 DEBUG [StoreCloser-TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/A/895ff6da1f74429ca4465d29c2017a8c to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/A/895ff6da1f74429ca4465d29c2017a8c 2024-11-26T19:16:21,482 DEBUG [StoreCloser-TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/B/12c9fed4ceb34602bb4b2c78e79553e8, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/B/b41d94fff91f46b6997e09703e8ee195, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/B/2e3d25928fd1481282c3d8b220a04607, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/B/d86ff56850bb415c845fe379281a2266, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/B/f3a61c66dfee4bd3b4626addc0a9137b, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/B/52d3b2ae440f4c778452b92b9f2525c0, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/B/4b8dac2818d8404c920ba4eac6e6537f, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/B/ede4d0b8067c4c95aa70e3390af1258d, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/B/070c70f9219049f68c31223fab166ff5, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/B/bb42897b36844730a4c0a3caf103b824, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/B/8463b575eec544c484e0ca2e5b672b70, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/B/1f75d62a2e074cbb87c33cb26fc32de0, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/B/af68b1e0375040e991f20f0372b67db1, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/B/1705406432154a82a6d671962c9d71a9, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/B/edaa78381ad947c7babd1d5e0c360134, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/B/d09ffbdba8f1471783d06d0177318ad5, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/B/e846ba6e090649c7a3b6d017d2d6262b, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/B/405151f3b7e64cce9610cf60729e30b8, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/B/6cbdc936a9f74a509a2f4af43bab250b, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/B/55bb29036d684ae394d321aef9f248cb, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/B/6f1d7b4d17c047e8b9039227133d9f5e, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/B/0e745d69292b4167aa3e14b40d38da5f, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/B/3d1f91045b2f4e9dbfd73617cf8ee12c] to archive 2024-11-26T19:16:21,483 DEBUG [StoreCloser-TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-26T19:16:21,484 DEBUG [StoreCloser-TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/B/12c9fed4ceb34602bb4b2c78e79553e8 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/B/12c9fed4ceb34602bb4b2c78e79553e8 2024-11-26T19:16:21,484 DEBUG [StoreCloser-TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/B/b41d94fff91f46b6997e09703e8ee195 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/B/b41d94fff91f46b6997e09703e8ee195 2024-11-26T19:16:21,485 DEBUG [StoreCloser-TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/B/2e3d25928fd1481282c3d8b220a04607 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/B/2e3d25928fd1481282c3d8b220a04607 2024-11-26T19:16:21,486 DEBUG [StoreCloser-TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/B/d86ff56850bb415c845fe379281a2266 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/B/d86ff56850bb415c845fe379281a2266 2024-11-26T19:16:21,487 DEBUG [StoreCloser-TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/B/f3a61c66dfee4bd3b4626addc0a9137b to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/B/f3a61c66dfee4bd3b4626addc0a9137b 2024-11-26T19:16:21,488 DEBUG [StoreCloser-TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/B/52d3b2ae440f4c778452b92b9f2525c0 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/B/52d3b2ae440f4c778452b92b9f2525c0 2024-11-26T19:16:21,488 DEBUG [StoreCloser-TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/B/4b8dac2818d8404c920ba4eac6e6537f to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/B/4b8dac2818d8404c920ba4eac6e6537f 2024-11-26T19:16:21,489 DEBUG [StoreCloser-TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/B/ede4d0b8067c4c95aa70e3390af1258d to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/B/ede4d0b8067c4c95aa70e3390af1258d 2024-11-26T19:16:21,489 DEBUG [StoreCloser-TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/B/070c70f9219049f68c31223fab166ff5 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/B/070c70f9219049f68c31223fab166ff5 2024-11-26T19:16:21,490 DEBUG [StoreCloser-TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/B/bb42897b36844730a4c0a3caf103b824 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/B/bb42897b36844730a4c0a3caf103b824 2024-11-26T19:16:21,491 DEBUG [StoreCloser-TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/B/8463b575eec544c484e0ca2e5b672b70 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/B/8463b575eec544c484e0ca2e5b672b70 2024-11-26T19:16:21,491 DEBUG [StoreCloser-TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/B/1f75d62a2e074cbb87c33cb26fc32de0 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/B/1f75d62a2e074cbb87c33cb26fc32de0 2024-11-26T19:16:21,492 DEBUG [StoreCloser-TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/B/af68b1e0375040e991f20f0372b67db1 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/B/af68b1e0375040e991f20f0372b67db1 2024-11-26T19:16:21,493 DEBUG [StoreCloser-TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/B/1705406432154a82a6d671962c9d71a9 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/B/1705406432154a82a6d671962c9d71a9 2024-11-26T19:16:21,493 DEBUG [StoreCloser-TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/B/edaa78381ad947c7babd1d5e0c360134 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/B/edaa78381ad947c7babd1d5e0c360134 2024-11-26T19:16:21,494 DEBUG [StoreCloser-TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/B/d09ffbdba8f1471783d06d0177318ad5 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/B/d09ffbdba8f1471783d06d0177318ad5 2024-11-26T19:16:21,494 DEBUG [StoreCloser-TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/B/e846ba6e090649c7a3b6d017d2d6262b to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/B/e846ba6e090649c7a3b6d017d2d6262b 2024-11-26T19:16:21,495 DEBUG [StoreCloser-TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/B/405151f3b7e64cce9610cf60729e30b8 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/B/405151f3b7e64cce9610cf60729e30b8 2024-11-26T19:16:21,496 DEBUG [StoreCloser-TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/B/6cbdc936a9f74a509a2f4af43bab250b to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/B/6cbdc936a9f74a509a2f4af43bab250b 2024-11-26T19:16:21,496 DEBUG [StoreCloser-TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/B/55bb29036d684ae394d321aef9f248cb to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/B/55bb29036d684ae394d321aef9f248cb 2024-11-26T19:16:21,497 DEBUG [StoreCloser-TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/B/6f1d7b4d17c047e8b9039227133d9f5e to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/B/6f1d7b4d17c047e8b9039227133d9f5e 2024-11-26T19:16:21,498 DEBUG [StoreCloser-TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/B/0e745d69292b4167aa3e14b40d38da5f to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/B/0e745d69292b4167aa3e14b40d38da5f 2024-11-26T19:16:21,498 DEBUG [StoreCloser-TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/B/3d1f91045b2f4e9dbfd73617cf8ee12c to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/B/3d1f91045b2f4e9dbfd73617cf8ee12c 2024-11-26T19:16:21,499 DEBUG [StoreCloser-TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/C/de6ca40c5e004b61a0dce3e14c1b8797, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/C/feb26fd9308a4719b36e217fc208430c, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/C/b359963e7f1045789fc2e91cfa41623b, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/C/d179307cb34c4305a24c407e63432847, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/C/703b372c747e455a80390069c05a1f02, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/C/a8c8a13ad4e04895a36aa6b3441a7216, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/C/2f8855040aa44190a8b22e9d3bd3757a, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/C/8f26cce1c3ac48c1879f93283371e670, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/C/b6d916657f044546aad705e55f69cb66, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/C/2dab618ba5a24c889d1162a4903b9fde, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/C/ca0549ea4d934b2ab9974943daf6d0c7, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/C/2bcc29443d4d42d9b3d43efafc67c907, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/C/8a4a90c8568a4137a69feb1f4566e551, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/C/49a99ab4394045bc857f9e20ac394ae6, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/C/5da694f9600d4813bb787d9cb753b75c, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/C/2595a75695074136a78f01b230e4891d, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/C/c7aee15270d0410fb440cc36174350f9, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/C/8cd22e96912c4ba49e35861c83070840, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/C/3337bedd73e542238f57cd4ceb1cf432, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/C/d68665aac38e4843be68c6d7ebb3b976, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/C/99b7a53251cd4c35980090a1724febae, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/C/913e09399393429ebb6f09340c30c1cb, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/C/36edecbc7e8742d7876f315d5896b46a] to archive 2024-11-26T19:16:21,500 DEBUG [StoreCloser-TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-26T19:16:21,501 DEBUG [StoreCloser-TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/C/de6ca40c5e004b61a0dce3e14c1b8797 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/C/de6ca40c5e004b61a0dce3e14c1b8797 2024-11-26T19:16:21,502 DEBUG [StoreCloser-TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/C/feb26fd9308a4719b36e217fc208430c to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/C/feb26fd9308a4719b36e217fc208430c 2024-11-26T19:16:21,502 DEBUG [StoreCloser-TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/C/b359963e7f1045789fc2e91cfa41623b to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/C/b359963e7f1045789fc2e91cfa41623b 2024-11-26T19:16:21,503 DEBUG [StoreCloser-TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/C/d179307cb34c4305a24c407e63432847 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/C/d179307cb34c4305a24c407e63432847 2024-11-26T19:16:21,504 DEBUG [StoreCloser-TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/C/703b372c747e455a80390069c05a1f02 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/C/703b372c747e455a80390069c05a1f02 2024-11-26T19:16:21,504 DEBUG [StoreCloser-TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/C/a8c8a13ad4e04895a36aa6b3441a7216 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/C/a8c8a13ad4e04895a36aa6b3441a7216 2024-11-26T19:16:21,505 DEBUG [StoreCloser-TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/C/2f8855040aa44190a8b22e9d3bd3757a to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/C/2f8855040aa44190a8b22e9d3bd3757a 2024-11-26T19:16:21,506 DEBUG [StoreCloser-TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/C/8f26cce1c3ac48c1879f93283371e670 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/C/8f26cce1c3ac48c1879f93283371e670 2024-11-26T19:16:21,507 DEBUG [StoreCloser-TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/C/b6d916657f044546aad705e55f69cb66 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/C/b6d916657f044546aad705e55f69cb66 2024-11-26T19:16:21,507 DEBUG [StoreCloser-TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/C/2dab618ba5a24c889d1162a4903b9fde to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/C/2dab618ba5a24c889d1162a4903b9fde 2024-11-26T19:16:21,508 DEBUG [StoreCloser-TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/C/ca0549ea4d934b2ab9974943daf6d0c7 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/C/ca0549ea4d934b2ab9974943daf6d0c7 2024-11-26T19:16:21,509 DEBUG [StoreCloser-TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/C/2bcc29443d4d42d9b3d43efafc67c907 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/C/2bcc29443d4d42d9b3d43efafc67c907 2024-11-26T19:16:21,509 DEBUG [StoreCloser-TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/C/8a4a90c8568a4137a69feb1f4566e551 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/C/8a4a90c8568a4137a69feb1f4566e551 2024-11-26T19:16:21,510 DEBUG [StoreCloser-TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/C/49a99ab4394045bc857f9e20ac394ae6 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/C/49a99ab4394045bc857f9e20ac394ae6 2024-11-26T19:16:21,511 DEBUG [StoreCloser-TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/C/5da694f9600d4813bb787d9cb753b75c to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/C/5da694f9600d4813bb787d9cb753b75c 2024-11-26T19:16:21,512 DEBUG [StoreCloser-TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/C/2595a75695074136a78f01b230e4891d to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/C/2595a75695074136a78f01b230e4891d 2024-11-26T19:16:21,512 DEBUG [StoreCloser-TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/C/c7aee15270d0410fb440cc36174350f9 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/C/c7aee15270d0410fb440cc36174350f9 2024-11-26T19:16:21,513 DEBUG [StoreCloser-TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/C/8cd22e96912c4ba49e35861c83070840 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/C/8cd22e96912c4ba49e35861c83070840 2024-11-26T19:16:21,514 DEBUG [StoreCloser-TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/C/3337bedd73e542238f57cd4ceb1cf432 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/C/3337bedd73e542238f57cd4ceb1cf432 2024-11-26T19:16:21,514 DEBUG [StoreCloser-TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/C/d68665aac38e4843be68c6d7ebb3b976 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/C/d68665aac38e4843be68c6d7ebb3b976 2024-11-26T19:16:21,515 DEBUG [StoreCloser-TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/C/99b7a53251cd4c35980090a1724febae to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/C/99b7a53251cd4c35980090a1724febae 2024-11-26T19:16:21,516 DEBUG [StoreCloser-TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/C/913e09399393429ebb6f09340c30c1cb to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/C/913e09399393429ebb6f09340c30c1cb 2024-11-26T19:16:21,516 DEBUG [StoreCloser-TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/C/36edecbc7e8742d7876f315d5896b46a to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/C/36edecbc7e8742d7876f315d5896b46a 2024-11-26T19:16:21,520 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/recovered.edits/510.seqid, newMaxSeqId=510, maxSeqId=1 2024-11-26T19:16:21,521 INFO [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. 2024-11-26T19:16:21,521 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1676): Region close journal for 741bfc4b4e2b320db573e476b103383f: Waiting for close lock at 1732648580225Running coprocessor pre-close hooks at 1732648580225Disabling compacts and flushes for region at 1732648580225Disabling writes for close at 1732648580225Obtaining lock to block concurrent updates at 1732648580225Preparing flush snapshotting stores in 741bfc4b4e2b320db573e476b103383f at 1732648580225Finished memstore snapshotting TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1732648580226 (+1 ms)Flushing stores of TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f. at 1732648580226Flushing 741bfc4b4e2b320db573e476b103383f/A: creating writer at 1732648580226Flushing 741bfc4b4e2b320db573e476b103383f/A: appending metadata at 1732648580228 (+2 ms)Flushing 741bfc4b4e2b320db573e476b103383f/A: closing flushed file at 1732648580228Flushing 741bfc4b4e2b320db573e476b103383f/B: creating writer at 1732648580635 (+407 ms)Flushing 741bfc4b4e2b320db573e476b103383f/B: appending metadata at 1732648580637 (+2 ms)Flushing 741bfc4b4e2b320db573e476b103383f/B: closing flushed file at 1732648580637Flushing 741bfc4b4e2b320db573e476b103383f/C: creating writer at 1732648581044 (+407 ms)Flushing 741bfc4b4e2b320db573e476b103383f/C: appending metadata at 1732648581046 (+2 ms)Flushing 741bfc4b4e2b320db573e476b103383f/C: closing flushed file at 1732648581046Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@36e63283: reopening flushed file at 1732648581453 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1a130710: reopening flushed file at 1732648581456 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@15082187: reopening flushed file at 1732648581459 (+3 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 741bfc4b4e2b320db573e476b103383f in 1237ms, sequenceid=507, compaction requested=true at 1732648581462 (+3 ms)Writing region close event to WAL at 1732648581518 (+56 ms)Running coprocessor post-close hooks at 1732648581521 (+3 ms)Closed at 1732648581521 2024-11-26T19:16:21,522 INFO [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] handler.UnassignRegionHandler(157): Closed 741bfc4b4e2b320db573e476b103383f 2024-11-26T19:16:21,523 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=135 updating hbase:meta row=741bfc4b4e2b320db573e476b103383f, regionState=CLOSED 2024-11-26T19:16:21,524 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=136, ppid=135, state=RUNNABLE, hasLock=false; CloseRegionProcedure 741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 because future has completed 2024-11-26T19:16:21,526 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-11-26T19:16:21,526 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; CloseRegionProcedure 741bfc4b4e2b320db573e476b103383f, server=fe76acf732b2,42785,1732648423970 in 1.4520 sec 2024-11-26T19:16:21,528 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=135, resume processing ppid=134 2024-11-26T19:16:21,528 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, ppid=134, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=741bfc4b4e2b320db573e476b103383f, UNASSIGN in 1.4560 sec 2024-11-26T19:16:21,529 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-11-26T19:16:21,529 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.4590 sec 2024-11-26T19:16:21,530 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732648581530"}]},"ts":"1732648581530"} 2024-11-26T19:16:21,531 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-26T19:16:21,531 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-26T19:16:21,533 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.4670 sec 2024-11-26T19:16:22,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-26T19:16:22,222 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-26T19:16:22,222 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-26T19:16:22,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] procedure2.ProcedureExecutor(1139): Stored pid=137, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T19:16:22,224 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=137, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T19:16:22,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-26T19:16:22,224 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=137, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T19:16:22,226 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f 2024-11-26T19:16:22,228 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/A, FileablePath, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/B, FileablePath, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/C, FileablePath, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/recovered.edits] 2024-11-26T19:16:22,230 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/A/17950f70b3a345168667fc1bbb56b274 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/A/17950f70b3a345168667fc1bbb56b274 2024-11-26T19:16:22,231 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/A/5b2ba53ee2d24b6da630a570c88e1573 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/A/5b2ba53ee2d24b6da630a570c88e1573 2024-11-26T19:16:22,232 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/A/baa49d8ee9cc442083ab97d211abbb62 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/A/baa49d8ee9cc442083ab97d211abbb62 2024-11-26T19:16:22,232 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/A/d63caad32301424fa4644cf18bca1f2c to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/A/d63caad32301424fa4644cf18bca1f2c 2024-11-26T19:16:22,234 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/B/04c777ab07e0436db08fde67d2420c8c to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/B/04c777ab07e0436db08fde67d2420c8c 2024-11-26T19:16:22,235 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/B/2bdab127d4694b859d037e9759a41353 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/B/2bdab127d4694b859d037e9759a41353 2024-11-26T19:16:22,236 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/B/40ebdb8eee39435eab3336deb50ddfd3 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/B/40ebdb8eee39435eab3336deb50ddfd3 2024-11-26T19:16:22,237 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/B/bdaf4b1b8d4a473e8d4fe56d99657983 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/B/bdaf4b1b8d4a473e8d4fe56d99657983 2024-11-26T19:16:22,238 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/C/13d1f43522664de7be30b89dd2053eb2 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/C/13d1f43522664de7be30b89dd2053eb2 2024-11-26T19:16:22,239 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/C/5baae9167df3467f82ab0da7f8a7ffa8 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/C/5baae9167df3467f82ab0da7f8a7ffa8 2024-11-26T19:16:22,240 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/C/7f30c26386fa402c9d9066bd9a79ce46 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/C/7f30c26386fa402c9d9066bd9a79ce46 2024-11-26T19:16:22,240 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/C/b0720d744826422182045cb6f81fab79 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/C/b0720d744826422182045cb6f81fab79 2024-11-26T19:16:22,242 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/recovered.edits/510.seqid to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f/recovered.edits/510.seqid 2024-11-26T19:16:22,243 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/741bfc4b4e2b320db573e476b103383f 2024-11-26T19:16:22,243 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-26T19:16:22,244 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=137, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T19:16:22,246 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-26T19:16:22,248 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-26T19:16:22,249 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=137, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T19:16:22,249 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-26T19:16:22,249 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732648582249"}]},"ts":"9223372036854775807"} 2024-11-26T19:16:22,250 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-26T19:16:22,250 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 741bfc4b4e2b320db573e476b103383f, NAME => 'TestAcidGuarantees,,1732648547142.741bfc4b4e2b320db573e476b103383f.', STARTKEY => '', ENDKEY => ''}] 2024-11-26T19:16:22,251 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-26T19:16:22,251 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732648582251"}]},"ts":"9223372036854775807"} 2024-11-26T19:16:22,252 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-26T19:16:22,252 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=137, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T19:16:22,253 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 30 msec 2024-11-26T19:16:22,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-26T19:16:22,332 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-26T19:16:22,333 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-26T19:16:22,344 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=246 (was 252), OpenFileDescriptor=441 (was 461), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=307 (was 354), ProcessCount=11 (was 11), AvailableMemoryMB=5695 (was 5763) 2024-11-26T19:16:22,352 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=246, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=307, ProcessCount=11, AvailableMemoryMB=5694 2024-11-26T19:16:22,353 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-26T19:16:22,354 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-26T19:16:22,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] procedure2.ProcedureExecutor(1139): Stored pid=138, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-26T19:16:22,355 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=138, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-26T19:16:22,355 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:22,355 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 138 2024-11-26T19:16:22,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-11-26T19:16:22,356 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=138, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-26T19:16:22,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742335_1511 (size=1061) 2024-11-26T19:16:22,362 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 66e19d6a9a9885e5b2c786f2e4c45a8e, NAME => 'TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16 2024-11-26T19:16:22,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742336_1512 (size=53) 2024-11-26T19:16:22,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-11-26T19:16:22,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-11-26T19:16:22,767 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T19:16:22,768 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 66e19d6a9a9885e5b2c786f2e4c45a8e, disabling compactions & flushes 2024-11-26T19:16:22,768 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e. 2024-11-26T19:16:22,768 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e. 2024-11-26T19:16:22,768 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e. after waiting 0 ms 2024-11-26T19:16:22,768 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e. 2024-11-26T19:16:22,768 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e. 2024-11-26T19:16:22,768 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 66e19d6a9a9885e5b2c786f2e4c45a8e: Waiting for close lock at 1732648582768Disabling compacts and flushes for region at 1732648582768Disabling writes for close at 1732648582768Writing region close event to WAL at 1732648582768Closed at 1732648582768 2024-11-26T19:16:22,769 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=138, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-26T19:16:22,769 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732648582769"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732648582769"}]},"ts":"1732648582769"} 2024-11-26T19:16:22,770 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-26T19:16:22,771 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=138, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-26T19:16:22,771 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732648582771"}]},"ts":"1732648582771"} 2024-11-26T19:16:22,772 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-26T19:16:22,772 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=66e19d6a9a9885e5b2c786f2e4c45a8e, ASSIGN}] 2024-11-26T19:16:22,773 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=66e19d6a9a9885e5b2c786f2e4c45a8e, ASSIGN 2024-11-26T19:16:22,774 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=66e19d6a9a9885e5b2c786f2e4c45a8e, ASSIGN; state=OFFLINE, location=fe76acf732b2,42785,1732648423970; forceNewPlan=false, retain=false 2024-11-26T19:16:22,924 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=139 updating hbase:meta row=66e19d6a9a9885e5b2c786f2e4c45a8e, regionState=OPENING, regionLocation=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:22,925 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=66e19d6a9a9885e5b2c786f2e4c45a8e, ASSIGN because future has completed 2024-11-26T19:16:22,926 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE, hasLock=false; OpenRegionProcedure 66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970}] 2024-11-26T19:16:22,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-11-26T19:16:23,080 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e. 2024-11-26T19:16:23,080 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(7752): Opening region: {ENCODED => 66e19d6a9a9885e5b2c786f2e4c45a8e, NAME => 'TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.', STARTKEY => '', ENDKEY => ''} 2024-11-26T19:16:23,081 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 66e19d6a9a9885e5b2c786f2e4c45a8e 2024-11-26T19:16:23,081 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T19:16:23,081 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(7794): checking encryption for 66e19d6a9a9885e5b2c786f2e4c45a8e 2024-11-26T19:16:23,081 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(7797): checking classloading for 66e19d6a9a9885e5b2c786f2e4c45a8e 2024-11-26T19:16:23,082 INFO [StoreOpener-66e19d6a9a9885e5b2c786f2e4c45a8e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 66e19d6a9a9885e5b2c786f2e4c45a8e 2024-11-26T19:16:23,083 INFO [StoreOpener-66e19d6a9a9885e5b2c786f2e4c45a8e-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-26T19:16:23,083 INFO [StoreOpener-66e19d6a9a9885e5b2c786f2e4c45a8e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 66e19d6a9a9885e5b2c786f2e4c45a8e columnFamilyName A 2024-11-26T19:16:23,083 DEBUG [StoreOpener-66e19d6a9a9885e5b2c786f2e4c45a8e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:23,083 INFO [StoreOpener-66e19d6a9a9885e5b2c786f2e4c45a8e-1 {}] regionserver.HStore(327): Store=66e19d6a9a9885e5b2c786f2e4c45a8e/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T19:16:23,083 INFO [StoreOpener-66e19d6a9a9885e5b2c786f2e4c45a8e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 66e19d6a9a9885e5b2c786f2e4c45a8e 2024-11-26T19:16:23,084 INFO [StoreOpener-66e19d6a9a9885e5b2c786f2e4c45a8e-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-26T19:16:23,084 INFO [StoreOpener-66e19d6a9a9885e5b2c786f2e4c45a8e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 66e19d6a9a9885e5b2c786f2e4c45a8e columnFamilyName B 2024-11-26T19:16:23,084 DEBUG [StoreOpener-66e19d6a9a9885e5b2c786f2e4c45a8e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:23,085 INFO [StoreOpener-66e19d6a9a9885e5b2c786f2e4c45a8e-1 {}] regionserver.HStore(327): Store=66e19d6a9a9885e5b2c786f2e4c45a8e/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T19:16:23,085 INFO [StoreOpener-66e19d6a9a9885e5b2c786f2e4c45a8e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 66e19d6a9a9885e5b2c786f2e4c45a8e 2024-11-26T19:16:23,086 INFO [StoreOpener-66e19d6a9a9885e5b2c786f2e4c45a8e-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-26T19:16:23,086 INFO [StoreOpener-66e19d6a9a9885e5b2c786f2e4c45a8e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 66e19d6a9a9885e5b2c786f2e4c45a8e columnFamilyName C 2024-11-26T19:16:23,086 DEBUG [StoreOpener-66e19d6a9a9885e5b2c786f2e4c45a8e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:23,086 INFO [StoreOpener-66e19d6a9a9885e5b2c786f2e4c45a8e-1 {}] regionserver.HStore(327): Store=66e19d6a9a9885e5b2c786f2e4c45a8e/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T19:16:23,086 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e. 2024-11-26T19:16:23,086 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1038): replaying wal for 66e19d6a9a9885e5b2c786f2e4c45a8e 2024-11-26T19:16:23,087 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e 2024-11-26T19:16:23,087 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e 2024-11-26T19:16:23,088 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1048): stopping wal replay for 66e19d6a9a9885e5b2c786f2e4c45a8e 2024-11-26T19:16:23,088 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1060): Cleaning up temporary data for 66e19d6a9a9885e5b2c786f2e4c45a8e 2024-11-26T19:16:23,088 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-26T19:16:23,089 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1093): writing seq id for 66e19d6a9a9885e5b2c786f2e4c45a8e 2024-11-26T19:16:23,090 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T19:16:23,091 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1114): Opened 66e19d6a9a9885e5b2c786f2e4c45a8e; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69905875, jitterRate=0.04167871177196503}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-26T19:16:23,091 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 66e19d6a9a9885e5b2c786f2e4c45a8e 2024-11-26T19:16:23,091 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1006): Region open journal for 66e19d6a9a9885e5b2c786f2e4c45a8e: Running coprocessor pre-open hook at 1732648583081Writing region info on filesystem at 1732648583081Initializing all the Stores at 1732648583081Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732648583081Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732648583082 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732648583082Cleaning up temporary data from old regions at 1732648583088 (+6 ms)Running coprocessor post-open hooks at 1732648583091 (+3 ms)Region opened successfully at 1732648583091 2024-11-26T19:16:23,092 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., pid=140, masterSystemTime=1732648583077 2024-11-26T19:16:23,094 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e. 2024-11-26T19:16:23,094 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e. 2024-11-26T19:16:23,094 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=139 updating hbase:meta row=66e19d6a9a9885e5b2c786f2e4c45a8e, regionState=OPEN, openSeqNum=2, regionLocation=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:23,095 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=140, ppid=139, state=RUNNABLE, hasLock=false; OpenRegionProcedure 66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 because future has completed 2024-11-26T19:16:23,097 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-11-26T19:16:23,097 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; OpenRegionProcedure 66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 in 170 msec 2024-11-26T19:16:23,099 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=139, resume processing ppid=138 2024-11-26T19:16:23,099 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, ppid=138, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=66e19d6a9a9885e5b2c786f2e4c45a8e, ASSIGN in 325 msec 2024-11-26T19:16:23,099 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=138, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-26T19:16:23,099 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732648583099"}]},"ts":"1732648583099"} 2024-11-26T19:16:23,100 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-26T19:16:23,101 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=138, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-26T19:16:23,102 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 747 msec 2024-11-26T19:16:23,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-11-26T19:16:23,492 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-26T19:16:23,493 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59939b53, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:16:23,493 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fe76acf732b2,33811,-1 for getting cluster id 2024-11-26T19:16:23,494 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T19:16:23,495 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3cec4e24-71db-4cd0-b63a-4ea30a9b160b' 2024-11-26T19:16:23,495 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T19:16:23,495 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3cec4e24-71db-4cd0-b63a-4ea30a9b160b" 2024-11-26T19:16:23,495 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24e7b09e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:16:23,495 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fe76acf732b2,33811,-1] 2024-11-26T19:16:23,495 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T19:16:23,495 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:16:23,496 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53246, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T19:16:23,497 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@340bc093, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:16:23,497 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T19:16:23,498 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fe76acf732b2,42785,1732648423970, seqNum=-1] 2024-11-26T19:16:23,498 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T19:16:23,498 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35400, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T19:16:23,499 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-26T19:16:23,500 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is fe76acf732b2,33811,1732648423099 2024-11-26T19:16:23,500 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@f295fe6 2024-11-26T19:16:23,500 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-26T19:16:23,500 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53256, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-26T19:16:23,501 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-26T19:16:23,502 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-26T19:16:23,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] procedure2.ProcedureExecutor(1139): Stored pid=141, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-26T19:16:23,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742337_1513 (size=1097) 2024-11-26T19:16:23,913 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-26T19:16:23,913 INFO [PEWorker-4 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-26T19:16:23,915 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=142, ppid=141, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-26T19:16:23,917 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=66e19d6a9a9885e5b2c786f2e4c45a8e, REOPEN/MOVE}] 2024-11-26T19:16:23,917 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=66e19d6a9a9885e5b2c786f2e4c45a8e, REOPEN/MOVE 2024-11-26T19:16:23,918 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=66e19d6a9a9885e5b2c786f2e4c45a8e, regionState=CLOSING, regionLocation=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:23,919 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=66e19d6a9a9885e5b2c786f2e4c45a8e, REOPEN/MOVE because future has completed 2024-11-26T19:16:23,919 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-26T19:16:23,919 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=144, ppid=143, state=RUNNABLE, hasLock=false; CloseRegionProcedure 66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970}] 2024-11-26T19:16:24,072 INFO [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] handler.UnassignRegionHandler(122): Close 66e19d6a9a9885e5b2c786f2e4c45a8e 2024-11-26T19:16:24,072 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-26T19:16:24,072 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1722): Closing 66e19d6a9a9885e5b2c786f2e4c45a8e, disabling compactions & flushes 2024-11-26T19:16:24,072 INFO [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e. 2024-11-26T19:16:24,072 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e. 2024-11-26T19:16:24,072 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e. after waiting 0 ms 2024-11-26T19:16:24,072 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e. 2024-11-26T19:16:24,081 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-26T19:16:24,082 INFO [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e. 2024-11-26T19:16:24,082 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1676): Region close journal for 66e19d6a9a9885e5b2c786f2e4c45a8e: Waiting for close lock at 1732648584072Running coprocessor pre-close hooks at 1732648584072Disabling compacts and flushes for region at 1732648584072Disabling writes for close at 1732648584072Writing region close event to WAL at 1732648584072Running coprocessor post-close hooks at 1732648584082 (+10 ms)Closed at 1732648584082 2024-11-26T19:16:24,082 WARN [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegionServer(3299): Not adding moved region record: 66e19d6a9a9885e5b2c786f2e4c45a8e to self. 2024-11-26T19:16:24,084 INFO [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] handler.UnassignRegionHandler(157): Closed 66e19d6a9a9885e5b2c786f2e4c45a8e 2024-11-26T19:16:24,085 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=66e19d6a9a9885e5b2c786f2e4c45a8e, regionState=CLOSED 2024-11-26T19:16:24,086 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=144, ppid=143, state=RUNNABLE, hasLock=false; CloseRegionProcedure 66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 because future has completed 2024-11-26T19:16:24,089 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=144, resume processing ppid=143 2024-11-26T19:16:24,089 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, ppid=143, state=SUCCESS, hasLock=false; CloseRegionProcedure 66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 in 168 msec 2024-11-26T19:16:24,090 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=66e19d6a9a9885e5b2c786f2e4c45a8e, REOPEN/MOVE; state=CLOSED, location=fe76acf732b2,42785,1732648423970; forceNewPlan=false, retain=true 2024-11-26T19:16:24,240 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=66e19d6a9a9885e5b2c786f2e4c45a8e, regionState=OPENING, regionLocation=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:24,242 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=66e19d6a9a9885e5b2c786f2e4c45a8e, REOPEN/MOVE because future has completed 2024-11-26T19:16:24,242 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=145, ppid=143, state=RUNNABLE, hasLock=false; OpenRegionProcedure 66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970}] 2024-11-26T19:16:24,396 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e. 2024-11-26T19:16:24,396 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(7752): Opening region: {ENCODED => 66e19d6a9a9885e5b2c786f2e4c45a8e, NAME => 'TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.', STARTKEY => '', ENDKEY => ''} 2024-11-26T19:16:24,396 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 66e19d6a9a9885e5b2c786f2e4c45a8e 2024-11-26T19:16:24,396 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T19:16:24,396 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(7794): checking encryption for 66e19d6a9a9885e5b2c786f2e4c45a8e 2024-11-26T19:16:24,396 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(7797): checking classloading for 66e19d6a9a9885e5b2c786f2e4c45a8e 2024-11-26T19:16:24,397 INFO [StoreOpener-66e19d6a9a9885e5b2c786f2e4c45a8e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 66e19d6a9a9885e5b2c786f2e4c45a8e 2024-11-26T19:16:24,398 INFO [StoreOpener-66e19d6a9a9885e5b2c786f2e4c45a8e-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-26T19:16:24,398 INFO [StoreOpener-66e19d6a9a9885e5b2c786f2e4c45a8e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 66e19d6a9a9885e5b2c786f2e4c45a8e columnFamilyName A 2024-11-26T19:16:24,399 DEBUG [StoreOpener-66e19d6a9a9885e5b2c786f2e4c45a8e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:24,399 INFO [StoreOpener-66e19d6a9a9885e5b2c786f2e4c45a8e-1 {}] regionserver.HStore(327): Store=66e19d6a9a9885e5b2c786f2e4c45a8e/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T19:16:24,400 INFO [StoreOpener-66e19d6a9a9885e5b2c786f2e4c45a8e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 66e19d6a9a9885e5b2c786f2e4c45a8e 2024-11-26T19:16:24,400 INFO [StoreOpener-66e19d6a9a9885e5b2c786f2e4c45a8e-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-26T19:16:24,400 INFO [StoreOpener-66e19d6a9a9885e5b2c786f2e4c45a8e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 66e19d6a9a9885e5b2c786f2e4c45a8e columnFamilyName B 2024-11-26T19:16:24,400 DEBUG [StoreOpener-66e19d6a9a9885e5b2c786f2e4c45a8e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:24,401 INFO [StoreOpener-66e19d6a9a9885e5b2c786f2e4c45a8e-1 {}] regionserver.HStore(327): Store=66e19d6a9a9885e5b2c786f2e4c45a8e/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T19:16:24,401 INFO [StoreOpener-66e19d6a9a9885e5b2c786f2e4c45a8e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 66e19d6a9a9885e5b2c786f2e4c45a8e 2024-11-26T19:16:24,401 INFO [StoreOpener-66e19d6a9a9885e5b2c786f2e4c45a8e-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-26T19:16:24,401 INFO [StoreOpener-66e19d6a9a9885e5b2c786f2e4c45a8e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 66e19d6a9a9885e5b2c786f2e4c45a8e columnFamilyName C 2024-11-26T19:16:24,401 DEBUG [StoreOpener-66e19d6a9a9885e5b2c786f2e4c45a8e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:24,402 INFO [StoreOpener-66e19d6a9a9885e5b2c786f2e4c45a8e-1 {}] regionserver.HStore(327): Store=66e19d6a9a9885e5b2c786f2e4c45a8e/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T19:16:24,402 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e. 2024-11-26T19:16:24,402 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1038): replaying wal for 66e19d6a9a9885e5b2c786f2e4c45a8e 2024-11-26T19:16:24,402 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e 2024-11-26T19:16:24,403 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e 2024-11-26T19:16:24,404 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1048): stopping wal replay for 66e19d6a9a9885e5b2c786f2e4c45a8e 2024-11-26T19:16:24,404 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1060): Cleaning up temporary data for 66e19d6a9a9885e5b2c786f2e4c45a8e 2024-11-26T19:16:24,404 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-26T19:16:24,405 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1093): writing seq id for 66e19d6a9a9885e5b2c786f2e4c45a8e 2024-11-26T19:16:24,405 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1114): Opened 66e19d6a9a9885e5b2c786f2e4c45a8e; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73936231, jitterRate=0.10173569619655609}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-26T19:16:24,405 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 66e19d6a9a9885e5b2c786f2e4c45a8e 2024-11-26T19:16:24,406 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1006): Region open journal for 66e19d6a9a9885e5b2c786f2e4c45a8e: Running coprocessor pre-open hook at 1732648584397Writing region info on filesystem at 1732648584397Initializing all the Stores at 1732648584397Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732648584397Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732648584397Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732648584397Cleaning up temporary data from old regions at 1732648584404 (+7 ms)Running coprocessor post-open hooks at 1732648584405 (+1 ms)Region opened successfully at 1732648584406 (+1 ms) 2024-11-26T19:16:24,407 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., pid=145, masterSystemTime=1732648584393 2024-11-26T19:16:24,408 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e. 2024-11-26T19:16:24,408 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e. 2024-11-26T19:16:24,409 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=66e19d6a9a9885e5b2c786f2e4c45a8e, regionState=OPEN, openSeqNum=5, regionLocation=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:24,410 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=145, ppid=143, state=RUNNABLE, hasLock=false; OpenRegionProcedure 66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 because future has completed 2024-11-26T19:16:24,411 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=145, resume processing ppid=143 2024-11-26T19:16:24,411 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, ppid=143, state=SUCCESS, hasLock=false; OpenRegionProcedure 66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 in 168 msec 2024-11-26T19:16:24,413 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=143, resume processing ppid=142 2024-11-26T19:16:24,413 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, ppid=142, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=66e19d6a9a9885e5b2c786f2e4c45a8e, REOPEN/MOVE in 495 msec 2024-11-26T19:16:24,414 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=142, resume processing ppid=141 2024-11-26T19:16:24,414 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, ppid=141, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 498 msec 2024-11-26T19:16:24,416 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 913 msec 2024-11-26T19:16:24,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-26T19:16:24,417 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-26T19:16:24,417 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30e225a5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:16:24,417 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fe76acf732b2,33811,-1 for getting cluster id 2024-11-26T19:16:24,418 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T19:16:24,418 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3cec4e24-71db-4cd0-b63a-4ea30a9b160b' 2024-11-26T19:16:24,418 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T19:16:24,419 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3cec4e24-71db-4cd0-b63a-4ea30a9b160b" 2024-11-26T19:16:24,419 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24ce3393, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:16:24,419 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fe76acf732b2,33811,-1] 2024-11-26T19:16:24,419 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T19:16:24,419 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:16:24,420 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53274, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T19:16:24,420 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a67ae6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:16:24,421 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3fea92ef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:16:24,421 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fe76acf732b2,33811,-1 for getting cluster id 2024-11-26T19:16:24,421 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T19:16:24,421 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3cec4e24-71db-4cd0-b63a-4ea30a9b160b' 2024-11-26T19:16:24,421 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T19:16:24,421 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3cec4e24-71db-4cd0-b63a-4ea30a9b160b" 2024-11-26T19:16:24,422 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d098f40, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:16:24,422 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fe76acf732b2,33811,-1] 2024-11-26T19:16:24,422 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T19:16:24,422 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:16:24,422 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53296, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T19:16:24,423 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b61a32e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:16:24,423 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c187dca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:16:24,423 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fe76acf732b2,33811,-1 for getting cluster id 2024-11-26T19:16:24,423 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T19:16:24,424 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3cec4e24-71db-4cd0-b63a-4ea30a9b160b' 2024-11-26T19:16:24,424 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T19:16:24,424 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3cec4e24-71db-4cd0-b63a-4ea30a9b160b" 2024-11-26T19:16:24,424 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ca7b5fd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:16:24,424 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fe76acf732b2,33811,-1] 2024-11-26T19:16:24,424 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T19:16:24,424 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:16:24,425 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53324, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T19:16:24,425 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4427adf3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:16:24,426 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@76ecb225, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:16:24,426 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fe76acf732b2,33811,-1 for getting cluster id 2024-11-26T19:16:24,426 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T19:16:24,426 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3cec4e24-71db-4cd0-b63a-4ea30a9b160b' 2024-11-26T19:16:24,427 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T19:16:24,427 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3cec4e24-71db-4cd0-b63a-4ea30a9b160b" 2024-11-26T19:16:24,427 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4eca7cfa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:16:24,427 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fe76acf732b2,33811,-1] 2024-11-26T19:16:24,427 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T19:16:24,427 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:16:24,427 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53330, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T19:16:24,428 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46ca97ce, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:16:24,428 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1407fbe0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:16:24,428 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fe76acf732b2,33811,-1 for getting cluster id 2024-11-26T19:16:24,429 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T19:16:24,429 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3cec4e24-71db-4cd0-b63a-4ea30a9b160b' 2024-11-26T19:16:24,429 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T19:16:24,429 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3cec4e24-71db-4cd0-b63a-4ea30a9b160b" 2024-11-26T19:16:24,429 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f8a86ee, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:16:24,429 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fe76acf732b2,33811,-1] 2024-11-26T19:16:24,429 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T19:16:24,429 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:16:24,430 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53350, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T19:16:24,430 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6aba7a6e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:16:24,431 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16d844d2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:16:24,431 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fe76acf732b2,33811,-1 for getting cluster id 2024-11-26T19:16:24,431 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T19:16:24,431 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3cec4e24-71db-4cd0-b63a-4ea30a9b160b' 2024-11-26T19:16:24,432 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T19:16:24,432 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3cec4e24-71db-4cd0-b63a-4ea30a9b160b" 2024-11-26T19:16:24,432 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bfc2fec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:16:24,432 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fe76acf732b2,33811,-1] 2024-11-26T19:16:24,432 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T19:16:24,432 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:16:24,432 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53356, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T19:16:24,433 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30c1029a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:16:24,433 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@533d9664, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:16:24,433 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fe76acf732b2,33811,-1 for getting cluster id 2024-11-26T19:16:24,433 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T19:16:24,434 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3cec4e24-71db-4cd0-b63a-4ea30a9b160b' 2024-11-26T19:16:24,434 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T19:16:24,434 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3cec4e24-71db-4cd0-b63a-4ea30a9b160b" 2024-11-26T19:16:24,434 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@667901d3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:16:24,434 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fe76acf732b2,33811,-1] 2024-11-26T19:16:24,434 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T19:16:24,434 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:16:24,435 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53376, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T19:16:24,435 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bc4e9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:16:24,436 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@383c5fef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:16:24,436 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fe76acf732b2,33811,-1 for getting cluster id 2024-11-26T19:16:24,436 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T19:16:24,437 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3cec4e24-71db-4cd0-b63a-4ea30a9b160b' 2024-11-26T19:16:24,437 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T19:16:24,437 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3cec4e24-71db-4cd0-b63a-4ea30a9b160b" 2024-11-26T19:16:24,437 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77566043, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:16:24,437 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fe76acf732b2,33811,-1] 2024-11-26T19:16:24,437 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T19:16:24,437 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:16:24,438 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53390, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T19:16:24,438 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c421a68, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:16:24,439 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b279a63, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:16:24,439 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fe76acf732b2,33811,-1 for getting cluster id 2024-11-26T19:16:24,439 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T19:16:24,440 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3cec4e24-71db-4cd0-b63a-4ea30a9b160b' 2024-11-26T19:16:24,440 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T19:16:24,440 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3cec4e24-71db-4cd0-b63a-4ea30a9b160b" 2024-11-26T19:16:24,440 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3df5f5b0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:16:24,440 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fe76acf732b2,33811,-1] 2024-11-26T19:16:24,440 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T19:16:24,441 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:16:24,441 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53416, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T19:16:24,441 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30588111, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:16:24,442 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10fb506a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:16:24,442 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fe76acf732b2,33811,-1 for getting cluster id 2024-11-26T19:16:24,442 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T19:16:24,442 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3cec4e24-71db-4cd0-b63a-4ea30a9b160b' 2024-11-26T19:16:24,442 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T19:16:24,442 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3cec4e24-71db-4cd0-b63a-4ea30a9b160b" 2024-11-26T19:16:24,443 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@477a11bf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:16:24,443 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fe76acf732b2,33811,-1] 2024-11-26T19:16:24,443 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T19:16:24,443 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:16:24,443 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53436, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T19:16:24,444 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63d681ce, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:16:24,444 DEBUG [Thread-2321 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T19:16:24,444 DEBUG [Thread-2302 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T19:16:24,444 DEBUG [Thread-2304 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T19:16:24,444 DEBUG [Thread-2315 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T19:16:24,445 DEBUG [Thread-2310 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T19:16:24,445 DEBUG [Thread-2317 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T19:16:24,445 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fe76acf732b2,42785,1732648423970, seqNum=-1] 2024-11-26T19:16:24,445 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T19:16:24,445 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fe76acf732b2,42785,1732648423970, seqNum=-1] 2024-11-26T19:16:24,445 DEBUG [Thread-2306 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T19:16:24,445 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fe76acf732b2,42785,1732648423970, seqNum=-1] 2024-11-26T19:16:24,445 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fe76acf732b2,42785,1732648423970, seqNum=-1] 2024-11-26T19:16:24,446 DEBUG [Thread-2308 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T19:16:24,446 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T19:16:24,446 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T19:16:24,446 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T19:16:24,446 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fe76acf732b2,42785,1732648423970, seqNum=-1] 2024-11-26T19:16:24,446 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T19:16:24,446 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35418, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T19:16:24,446 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T19:16:24,447 DEBUG [Thread-2313 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T19:16:24,447 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35426, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T19:16:24,447 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35438, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T19:16:24,447 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35442, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T19:16:24,447 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fe76acf732b2,42785,1732648423970, seqNum=-1] 2024-11-26T19:16:24,447 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T19:16:24,447 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35416, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T19:16:24,447 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fe76acf732b2,42785,1732648423970, seqNum=-1] 2024-11-26T19:16:24,448 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T19:16:24,448 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fe76acf732b2,42785,1732648423970, seqNum=-1] 2024-11-26T19:16:24,448 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35454, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T19:16:24,448 DEBUG [Thread-2319 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T19:16:24,448 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T19:16:24,449 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5] 2024-11-26T19:16:24,449 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fe76acf732b2,42785,1732648423970, seqNum=-1] 2024-11-26T19:16:24,449 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T19:16:24,449 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5] 2024-11-26T19:16:24,449 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5] 2024-11-26T19:16:24,450 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35466, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T19:16:24,450 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35478, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T19:16:24,450 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5] 2024-11-26T19:16:24,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] procedure2.ProcedureExecutor(1139): Stored pid=146, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees 2024-11-26T19:16:24,451 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35456, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T19:16:24,451 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5] 2024-11-26T19:16:24,451 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5] 2024-11-26T19:16:24,452 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=146, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T19:16:24,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-26T19:16:24,452 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5] 2024-11-26T19:16:24,453 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fe76acf732b2,42785,1732648423970, seqNum=-1] 2024-11-26T19:16:24,453 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5] 2024-11-26T19:16:24,453 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T19:16:24,453 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5] 2024-11-26T19:16:24,454 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=146, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T19:16:24,454 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=146, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T19:16:24,454 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35492, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T19:16:24,456 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5] 2024-11-26T19:16:24,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(8855): Flush requested on 66e19d6a9a9885e5b2c786f2e4c45a8e 2024-11-26T19:16:24,458 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 66e19d6a9a9885e5b2c786f2e4c45a8e 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-26T19:16:24,459 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66e19d6a9a9885e5b2c786f2e4c45a8e, store=A 2024-11-26T19:16:24,459 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:16:24,459 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66e19d6a9a9885e5b2c786f2e4c45a8e, store=B 2024-11-26T19:16:24,459 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:16:24,459 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66e19d6a9a9885e5b2c786f2e4c45a8e, store=C 2024-11-26T19:16:24,459 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:16:24,479 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126ca2b373b22254a5f8dd790763ef52acf_66e19d6a9a9885e5b2c786f2e4c45a8e is 50, key is test_row_0/A:col10/1732648584458/Put/seqid=0 2024-11-26T19:16:24,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742338_1514 (size=14594) 2024-11-26T19:16:24,516 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:24,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35456 deadline: 1732648644509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:24,516 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:24,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35438 deadline: 1732648644510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:24,516 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:24,516 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:24,516 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:24,516 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:24,516 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:24,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35454 deadline: 1732648644512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:24,516 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:24,516 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:24,517 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:24,517 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:24,517 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:24,517 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:24,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35416 deadline: 1732648644513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:24,517 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:24,518 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:24,518 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:24,518 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:24,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35426 deadline: 1732648644514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:24,518 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:24,518 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:24,518 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:24,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-26T19:16:24,608 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-26T19:16:24,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e. 2024-11-26T19:16:24,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e. as already flushing 2024-11-26T19:16:24,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e. 2024-11-26T19:16:24,609 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 66e19d6a9a9885e5b2c786f2e4c45a8e, NAME => 'TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:16:24,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 66e19d6a9a9885e5b2c786f2e4c45a8e, NAME => 'TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:16:24,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66e19d6a9a9885e5b2c786f2e4c45a8e, NAME => 'TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66e19d6a9a9885e5b2c786f2e4c45a8e, NAME => 'TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:16:24,625 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:24,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35454 deadline: 1732648644622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:24,625 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:24,625 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:24,625 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:24,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35426 deadline: 1732648644622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:24,625 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:24,625 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:24,625 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:24,626 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:24,626 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:24,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35456 deadline: 1732648644622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:24,626 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:24,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35438 deadline: 1732648644622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:24,626 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:24,626 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:24,626 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:24,626 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:24,626 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:24,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35416 deadline: 1732648644622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:24,626 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:24,626 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:24,626 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:24,626 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:24,626 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:24,761 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-26T19:16:24,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e. 2024-11-26T19:16:24,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e. as already flushing 2024-11-26T19:16:24,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e. 2024-11-26T19:16:24,762 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 66e19d6a9a9885e5b2c786f2e4c45a8e, NAME => 'TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:16:24,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 66e19d6a9a9885e5b2c786f2e4c45a8e, NAME => 'TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:16:24,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66e19d6a9a9885e5b2c786f2e4c45a8e, NAME => 'TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66e19d6a9a9885e5b2c786f2e4c45a8e, NAME => 'TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:16:24,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-26T19:16:24,834 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:24,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35454 deadline: 1732648644831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:24,835 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:24,835 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:24,835 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:24,835 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:24,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35426 deadline: 1732648644832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:24,835 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:24,835 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:24,835 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:24,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35456 deadline: 1732648644832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:24,835 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:24,835 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:24,836 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:24,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35438 deadline: 1732648644832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:24,836 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:24,836 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:24,836 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:24,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35416 deadline: 1732648644832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:24,836 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:24,836 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:24,836 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:24,836 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:24,836 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:24,836 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:24,883 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:24,887 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126ca2b373b22254a5f8dd790763ef52acf_66e19d6a9a9885e5b2c786f2e4c45a8e to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126ca2b373b22254a5f8dd790763ef52acf_66e19d6a9a9885e5b2c786f2e4c45a8e 2024-11-26T19:16:24,888 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/A/87a642e497174afca90efe8d898f0b53, store: [table=TestAcidGuarantees family=A region=66e19d6a9a9885e5b2c786f2e4c45a8e] 2024-11-26T19:16:24,888 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/A/87a642e497174afca90efe8d898f0b53 is 175, key is test_row_0/A:col10/1732648584458/Put/seqid=0 2024-11-26T19:16:24,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742339_1515 (size=39549) 2024-11-26T19:16:24,914 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-26T19:16:24,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e. 2024-11-26T19:16:24,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e. as already flushing 2024-11-26T19:16:24,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e. 2024-11-26T19:16:24,915 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 66e19d6a9a9885e5b2c786f2e4c45a8e, NAME => 'TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:16:24,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 66e19d6a9a9885e5b2c786f2e4c45a8e, NAME => 'TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:16:24,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66e19d6a9a9885e5b2c786f2e4c45a8e, NAME => 'TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66e19d6a9a9885e5b2c786f2e4c45a8e, NAME => 'TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:16:25,067 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-26T19:16:25,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e. 2024-11-26T19:16:25,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e. as already flushing 2024-11-26T19:16:25,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e. 2024-11-26T19:16:25,068 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 66e19d6a9a9885e5b2c786f2e4c45a8e, NAME => 'TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:16:25,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 66e19d6a9a9885e5b2c786f2e4c45a8e, NAME => 'TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:16:25,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66e19d6a9a9885e5b2c786f2e4c45a8e, NAME => 'TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66e19d6a9a9885e5b2c786f2e4c45a8e, NAME => 'TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:16:25,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-26T19:16:25,144 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:25,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35454 deadline: 1732648645141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:25,145 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:25,145 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:25,145 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:25,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35426 deadline: 1732648645142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:25,145 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:25,145 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:25,145 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:25,145 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:25,145 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:25,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35456 deadline: 1732648645142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:25,146 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:25,145 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:25,146 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:25,146 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:25,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35416 deadline: 1732648645142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:25,146 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:25,146 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:25,146 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:25,146 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:25,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35438 deadline: 1732648645142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:25,146 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:25,146 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:25,146 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:25,220 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-26T19:16:25,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e. 2024-11-26T19:16:25,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e. as already flushing 2024-11-26T19:16:25,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e. 2024-11-26T19:16:25,220 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 66e19d6a9a9885e5b2c786f2e4c45a8e, NAME => 'TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:16:25,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 66e19d6a9a9885e5b2c786f2e4c45a8e, NAME => 'TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:16:25,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66e19d6a9a9885e5b2c786f2e4c45a8e, NAME => 'TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66e19d6a9a9885e5b2c786f2e4c45a8e, NAME => 'TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:16:25,301 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=17, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/A/87a642e497174afca90efe8d898f0b53 2024-11-26T19:16:25,328 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/B/9da0a6ee2caf44cfbf4ffbdea845fc85 is 50, key is test_row_0/B:col10/1732648584458/Put/seqid=0 2024-11-26T19:16:25,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742340_1516 (size=12001) 2024-11-26T19:16:25,334 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/B/9da0a6ee2caf44cfbf4ffbdea845fc85 2024-11-26T19:16:25,360 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/C/5c228e0d704a4e819dd191c5b945755c is 50, key is test_row_0/C:col10/1732648584458/Put/seqid=0 2024-11-26T19:16:25,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742341_1517 (size=12001) 2024-11-26T19:16:25,373 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-26T19:16:25,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e. 2024-11-26T19:16:25,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e. as already flushing 2024-11-26T19:16:25,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e. 2024-11-26T19:16:25,373 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 66e19d6a9a9885e5b2c786f2e4c45a8e, NAME => 'TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:16:25,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 66e19d6a9a9885e5b2c786f2e4c45a8e, NAME => 'TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:16:25,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66e19d6a9a9885e5b2c786f2e4c45a8e, NAME => 'TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66e19d6a9a9885e5b2c786f2e4c45a8e, NAME => 'TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:16:25,526 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-26T19:16:25,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e. 2024-11-26T19:16:25,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e. as already flushing 2024-11-26T19:16:25,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e. 2024-11-26T19:16:25,526 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 66e19d6a9a9885e5b2c786f2e4c45a8e, NAME => 'TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:16:25,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 66e19d6a9a9885e5b2c786f2e4c45a8e, NAME => 'TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:16:25,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66e19d6a9a9885e5b2c786f2e4c45a8e, NAME => 'TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66e19d6a9a9885e5b2c786f2e4c45a8e, NAME => 'TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:16:25,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-26T19:16:25,655 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:25,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35454 deadline: 1732648645652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:25,656 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:25,656 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:25,656 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:25,656 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:25,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35426 deadline: 1732648645652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:25,656 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:25,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35456 deadline: 1732648645652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:25,656 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:25,656 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:25,656 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:25,656 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:25,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35416 deadline: 1732648645652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:25,657 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:25,657 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:25,657 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:25,657 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:25,657 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:25,657 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:25,662 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:25,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35438 deadline: 1732648645662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:25,662 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:25,662 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:25,662 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:25,679 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-26T19:16:25,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e. 2024-11-26T19:16:25,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e. as already flushing 2024-11-26T19:16:25,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e. 2024-11-26T19:16:25,679 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 66e19d6a9a9885e5b2c786f2e4c45a8e, NAME => 'TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:16:25,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 66e19d6a9a9885e5b2c786f2e4c45a8e, NAME => 'TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:16:25,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66e19d6a9a9885e5b2c786f2e4c45a8e, NAME => 'TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66e19d6a9a9885e5b2c786f2e4c45a8e, NAME => 'TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:16:25,764 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/C/5c228e0d704a4e819dd191c5b945755c 2024-11-26T19:16:25,768 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/A/87a642e497174afca90efe8d898f0b53 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/87a642e497174afca90efe8d898f0b53 2024-11-26T19:16:25,772 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/87a642e497174afca90efe8d898f0b53, entries=200, sequenceid=17, filesize=38.6 K 2024-11-26T19:16:25,772 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/B/9da0a6ee2caf44cfbf4ffbdea845fc85 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/B/9da0a6ee2caf44cfbf4ffbdea845fc85 2024-11-26T19:16:25,776 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/B/9da0a6ee2caf44cfbf4ffbdea845fc85, entries=150, sequenceid=17, filesize=11.7 K 2024-11-26T19:16:25,776 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/C/5c228e0d704a4e819dd191c5b945755c as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/C/5c228e0d704a4e819dd191c5b945755c 2024-11-26T19:16:25,780 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/C/5c228e0d704a4e819dd191c5b945755c, entries=150, sequenceid=17, filesize=11.7 K 2024-11-26T19:16:25,780 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 66e19d6a9a9885e5b2c786f2e4c45a8e in 1322ms, sequenceid=17, compaction requested=false 2024-11-26T19:16:25,780 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-26T19:16:25,781 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 66e19d6a9a9885e5b2c786f2e4c45a8e: 2024-11-26T19:16:25,832 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-26T19:16:25,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e. 2024-11-26T19:16:25,832 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2902): Flushing 66e19d6a9a9885e5b2c786f2e4c45a8e 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-26T19:16:25,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66e19d6a9a9885e5b2c786f2e4c45a8e, store=A 2024-11-26T19:16:25,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:16:25,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66e19d6a9a9885e5b2c786f2e4c45a8e, store=B 2024-11-26T19:16:25,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:16:25,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66e19d6a9a9885e5b2c786f2e4c45a8e, store=C 2024-11-26T19:16:25,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:16:25,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411261e67226e811a4483a6b73cdc2fe0c646_66e19d6a9a9885e5b2c786f2e4c45a8e is 50, key is test_row_0/A:col10/1732648584512/Put/seqid=0 2024-11-26T19:16:25,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742342_1518 (size=12154) 2024-11-26T19:16:25,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:25,860 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411261e67226e811a4483a6b73cdc2fe0c646_66e19d6a9a9885e5b2c786f2e4c45a8e to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411261e67226e811a4483a6b73cdc2fe0c646_66e19d6a9a9885e5b2c786f2e4c45a8e 2024-11-26T19:16:25,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/A/b157fb2cef044d1898092359ba36acc0, store: [table=TestAcidGuarantees family=A region=66e19d6a9a9885e5b2c786f2e4c45a8e] 2024-11-26T19:16:25,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/A/b157fb2cef044d1898092359ba36acc0 is 175, key is test_row_0/A:col10/1732648584512/Put/seqid=0 2024-11-26T19:16:25,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742343_1519 (size=30955) 2024-11-26T19:16:26,265 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=36, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/A/b157fb2cef044d1898092359ba36acc0 2024-11-26T19:16:26,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/B/74374299f8d342ef886cce0569e0d824 is 50, key is test_row_0/B:col10/1732648584512/Put/seqid=0 2024-11-26T19:16:26,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742344_1520 (size=12001) 2024-11-26T19:16:26,276 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=36 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/B/74374299f8d342ef886cce0569e0d824 2024-11-26T19:16:26,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/C/0fc42d0edd174075bd86848d968acc8f is 50, key is test_row_0/C:col10/1732648584512/Put/seqid=0 2024-11-26T19:16:26,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742345_1521 (size=12001) 2024-11-26T19:16:26,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-26T19:16:26,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(8855): Flush requested on 66e19d6a9a9885e5b2c786f2e4c45a8e 2024-11-26T19:16:26,674 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e. as already flushing 2024-11-26T19:16:26,684 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:26,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35454 deadline: 1732648646680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:26,685 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:26,685 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:26,685 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:26,686 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:26,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35426 deadline: 1732648646682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:26,686 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:26,686 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:26,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35456 deadline: 1732648646683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:26,686 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:26,686 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:26,686 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:26,686 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:26,686 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:26,686 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:26,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35416 deadline: 1732648646684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:26,687 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:26,687 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:26,687 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:26,689 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=36 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/C/0fc42d0edd174075bd86848d968acc8f 2024-11-26T19:16:26,690 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:26,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35438 deadline: 1732648646685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:26,690 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:26,690 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:26,690 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:26,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/A/b157fb2cef044d1898092359ba36acc0 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/b157fb2cef044d1898092359ba36acc0 2024-11-26T19:16:26,702 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/b157fb2cef044d1898092359ba36acc0, entries=150, sequenceid=36, filesize=30.2 K 2024-11-26T19:16:26,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/B/74374299f8d342ef886cce0569e0d824 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/B/74374299f8d342ef886cce0569e0d824 2024-11-26T19:16:26,706 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/B/74374299f8d342ef886cce0569e0d824, entries=150, sequenceid=36, filesize=11.7 K 2024-11-26T19:16:26,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/C/0fc42d0edd174075bd86848d968acc8f as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/C/0fc42d0edd174075bd86848d968acc8f 2024-11-26T19:16:26,710 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/C/0fc42d0edd174075bd86848d968acc8f, entries=150, sequenceid=36, filesize=11.7 K 2024-11-26T19:16:26,710 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 66e19d6a9a9885e5b2c786f2e4c45a8e in 878ms, sequenceid=36, compaction requested=false 2024-11-26T19:16:26,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2603): Flush status journal for 66e19d6a9a9885e5b2c786f2e4c45a8e: 2024-11-26T19:16:26,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e. 2024-11-26T19:16:26,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=147 2024-11-26T19:16:26,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster(4169): Remote procedure done, pid=147 2024-11-26T19:16:26,714 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=146 2024-11-26T19:16:26,714 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=146, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2580 sec 2024-11-26T19:16:26,716 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees in 2.2670 sec 2024-11-26T19:16:26,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(8855): Flush requested on 66e19d6a9a9885e5b2c786f2e4c45a8e 2024-11-26T19:16:26,795 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 66e19d6a9a9885e5b2c786f2e4c45a8e 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-26T19:16:26,796 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66e19d6a9a9885e5b2c786f2e4c45a8e, store=A 2024-11-26T19:16:26,796 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:16:26,796 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66e19d6a9a9885e5b2c786f2e4c45a8e, store=B 2024-11-26T19:16:26,796 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:16:26,796 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66e19d6a9a9885e5b2c786f2e4c45a8e, store=C 2024-11-26T19:16:26,796 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:16:26,807 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126c8cc880d47a3449797351f44efc2fd1c_66e19d6a9a9885e5b2c786f2e4c45a8e is 50, key is test_row_0/A:col10/1732648586793/Put/seqid=0 2024-11-26T19:16:26,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742346_1522 (size=12154) 2024-11-26T19:16:26,812 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:26,815 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126c8cc880d47a3449797351f44efc2fd1c_66e19d6a9a9885e5b2c786f2e4c45a8e to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126c8cc880d47a3449797351f44efc2fd1c_66e19d6a9a9885e5b2c786f2e4c45a8e 2024-11-26T19:16:26,816 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/A/1348ef0ddc8e48d1a99546dee9f1b03a, store: [table=TestAcidGuarantees family=A region=66e19d6a9a9885e5b2c786f2e4c45a8e] 2024-11-26T19:16:26,816 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/A/1348ef0ddc8e48d1a99546dee9f1b03a is 175, key is test_row_0/A:col10/1732648586793/Put/seqid=0 2024-11-26T19:16:26,820 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:26,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35416 deadline: 1732648646812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:26,820 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:26,820 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:26,820 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:26,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35438 deadline: 1732648646813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:26,820 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:26,821 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:26,821 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:26,821 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:26,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742347_1523 (size=30955) 2024-11-26T19:16:26,827 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:26,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35454 deadline: 1732648646819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:26,827 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:26,827 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:26,827 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:26,827 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:26,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35426 deadline: 1732648646821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:26,827 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:26,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35456 deadline: 1732648646820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:26,827 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:26,828 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:26,828 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:26,828 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:26,828 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:26,828 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:26,933 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:26,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35416 deadline: 1732648646932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:26,934 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:26,934 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:26,934 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:26,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35438 deadline: 1732648646932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:26,934 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:26,934 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:26,934 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:26,934 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:26,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35454 deadline: 1732648646932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:26,934 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:26,934 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:26,934 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:26,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35426 deadline: 1732648646932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:26,934 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:26,934 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:26,934 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:26,935 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:26,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35456 deadline: 1732648646932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:26,935 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:26,935 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:26,935 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:26,935 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:26,935 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:27,040 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-26T19:16:27,143 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:27,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35416 deadline: 1732648647142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:27,143 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:27,143 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:27,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35438 deadline: 1732648647142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:27,144 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:27,144 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:27,144 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:27,144 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:27,144 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:27,144 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:27,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35454 deadline: 1732648647142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:27,144 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:27,144 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:27,144 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:27,146 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:27,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35426 deadline: 1732648647142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:27,146 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:27,146 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:27,146 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:27,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35456 deadline: 1732648647142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:27,146 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:27,147 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:27,147 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:27,147 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:27,226 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=52, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/A/1348ef0ddc8e48d1a99546dee9f1b03a 2024-11-26T19:16:27,233 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/B/ebff55f6c85147a8b651e7722a0d7c2c is 50, key is test_row_0/B:col10/1732648586793/Put/seqid=0 2024-11-26T19:16:27,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742348_1524 (size=12001) 2024-11-26T19:16:27,455 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:27,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35416 deadline: 1732648647452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:27,456 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:27,456 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:27,456 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:27,456 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:27,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35438 deadline: 1732648647452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:27,456 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:27,456 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:27,456 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:27,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35454 deadline: 1732648647452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:27,456 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:27,456 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:27,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35426 deadline: 1732648647452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:27,456 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:27,457 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:27,457 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:27,457 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:27,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35456 deadline: 1732648647452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:27,457 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:27,457 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:27,457 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:27,457 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:27,457 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:27,457 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:27,637 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/B/ebff55f6c85147a8b651e7722a0d7c2c 2024-11-26T19:16:27,645 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/C/484760cc712349a5adda6432ccd0f622 is 50, key is test_row_0/C:col10/1732648586793/Put/seqid=0 2024-11-26T19:16:27,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742349_1525 (size=12001) 2024-11-26T19:16:27,648 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/C/484760cc712349a5adda6432ccd0f622 2024-11-26T19:16:27,652 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/A/1348ef0ddc8e48d1a99546dee9f1b03a as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/1348ef0ddc8e48d1a99546dee9f1b03a 2024-11-26T19:16:27,655 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/1348ef0ddc8e48d1a99546dee9f1b03a, entries=150, sequenceid=52, filesize=30.2 K 2024-11-26T19:16:27,656 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/B/ebff55f6c85147a8b651e7722a0d7c2c as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/B/ebff55f6c85147a8b651e7722a0d7c2c 2024-11-26T19:16:27,659 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/B/ebff55f6c85147a8b651e7722a0d7c2c, entries=150, sequenceid=52, filesize=11.7 K 2024-11-26T19:16:27,659 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/C/484760cc712349a5adda6432ccd0f622 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/C/484760cc712349a5adda6432ccd0f622 2024-11-26T19:16:27,663 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/C/484760cc712349a5adda6432ccd0f622, entries=150, sequenceid=52, filesize=11.7 K 2024-11-26T19:16:27,664 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=93.93 KB/96180 for 66e19d6a9a9885e5b2c786f2e4c45a8e in 869ms, sequenceid=52, compaction requested=true 2024-11-26T19:16:27,664 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 66e19d6a9a9885e5b2c786f2e4c45a8e: 2024-11-26T19:16:27,664 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 66e19d6a9a9885e5b2c786f2e4c45a8e:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T19:16:27,664 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:16:27,664 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T19:16:27,664 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T19:16:27,664 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 66e19d6a9a9885e5b2c786f2e4c45a8e:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T19:16:27,664 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:16:27,665 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 66e19d6a9a9885e5b2c786f2e4c45a8e:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T19:16:27,665 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T19:16:27,666 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T19:16:27,666 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T19:16:27,666 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1541): 66e19d6a9a9885e5b2c786f2e4c45a8e/A is initiating minor compaction (all files) 2024-11-26T19:16:27,666 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1541): 66e19d6a9a9885e5b2c786f2e4c45a8e/B is initiating minor compaction (all files) 2024-11-26T19:16:27,666 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 66e19d6a9a9885e5b2c786f2e4c45a8e/A in TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e. 2024-11-26T19:16:27,666 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 66e19d6a9a9885e5b2c786f2e4c45a8e/B in TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e. 2024-11-26T19:16:27,666 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/B/9da0a6ee2caf44cfbf4ffbdea845fc85, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/B/74374299f8d342ef886cce0569e0d824, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/B/ebff55f6c85147a8b651e7722a0d7c2c] into tmpdir=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp, totalSize=35.2 K 2024-11-26T19:16:27,667 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/87a642e497174afca90efe8d898f0b53, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/b157fb2cef044d1898092359ba36acc0, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/1348ef0ddc8e48d1a99546dee9f1b03a] into tmpdir=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp, totalSize=99.1 K 2024-11-26T19:16:27,667 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e. 2024-11-26T19:16:27,667 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e. files: [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/87a642e497174afca90efe8d898f0b53, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/b157fb2cef044d1898092359ba36acc0, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/1348ef0ddc8e48d1a99546dee9f1b03a] 2024-11-26T19:16:27,667 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting 9da0a6ee2caf44cfbf4ffbdea845fc85, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1732648584454 2024-11-26T19:16:27,667 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting 87a642e497174afca90efe8d898f0b53, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1732648584454 2024-11-26T19:16:27,667 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting 74374299f8d342ef886cce0569e0d824, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1732648584510 2024-11-26T19:16:27,667 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting b157fb2cef044d1898092359ba36acc0, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1732648584510 2024-11-26T19:16:27,668 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1348ef0ddc8e48d1a99546dee9f1b03a, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732648586683 2024-11-26T19:16:27,668 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting ebff55f6c85147a8b651e7722a0d7c2c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732648586683 2024-11-26T19:16:27,673 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=66e19d6a9a9885e5b2c786f2e4c45a8e] 2024-11-26T19:16:27,675 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 66e19d6a9a9885e5b2c786f2e4c45a8e#B#compaction#447 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T19:16:27,675 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/B/a2569dff74ba4043b514760294e8b147 is 50, key is test_row_0/B:col10/1732648586793/Put/seqid=0 2024-11-26T19:16:27,676 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112645416229c6434520a1f4beccc5b96a28_66e19d6a9a9885e5b2c786f2e4c45a8e store=[table=TestAcidGuarantees family=A region=66e19d6a9a9885e5b2c786f2e4c45a8e] 2024-11-26T19:16:27,677 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112645416229c6434520a1f4beccc5b96a28_66e19d6a9a9885e5b2c786f2e4c45a8e, store=[table=TestAcidGuarantees family=A region=66e19d6a9a9885e5b2c786f2e4c45a8e] 2024-11-26T19:16:27,677 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112645416229c6434520a1f4beccc5b96a28_66e19d6a9a9885e5b2c786f2e4c45a8e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=66e19d6a9a9885e5b2c786f2e4c45a8e] 2024-11-26T19:16:27,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742350_1526 (size=12104) 2024-11-26T19:16:27,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742351_1527 (size=4469) 2024-11-26T19:16:27,689 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/B/a2569dff74ba4043b514760294e8b147 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/B/a2569dff74ba4043b514760294e8b147 2024-11-26T19:16:27,694 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 66e19d6a9a9885e5b2c786f2e4c45a8e/B of 66e19d6a9a9885e5b2c786f2e4c45a8e into a2569dff74ba4043b514760294e8b147(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T19:16:27,694 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 66e19d6a9a9885e5b2c786f2e4c45a8e: 2024-11-26T19:16:27,694 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., storeName=66e19d6a9a9885e5b2c786f2e4c45a8e/B, priority=13, startTime=1732648587664; duration=0sec 2024-11-26T19:16:27,694 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T19:16:27,694 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 66e19d6a9a9885e5b2c786f2e4c45a8e:B 2024-11-26T19:16:27,694 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T19:16:27,695 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T19:16:27,695 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1541): 66e19d6a9a9885e5b2c786f2e4c45a8e/C is initiating minor compaction (all files) 2024-11-26T19:16:27,695 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 66e19d6a9a9885e5b2c786f2e4c45a8e/C in TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e. 2024-11-26T19:16:27,695 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/C/5c228e0d704a4e819dd191c5b945755c, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/C/0fc42d0edd174075bd86848d968acc8f, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/C/484760cc712349a5adda6432ccd0f622] into tmpdir=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp, totalSize=35.2 K 2024-11-26T19:16:27,695 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting 5c228e0d704a4e819dd191c5b945755c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1732648584454 2024-11-26T19:16:27,695 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting 0fc42d0edd174075bd86848d968acc8f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1732648584510 2024-11-26T19:16:27,696 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting 484760cc712349a5adda6432ccd0f622, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732648586683 2024-11-26T19:16:27,702 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 66e19d6a9a9885e5b2c786f2e4c45a8e#C#compaction#448 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T19:16:27,702 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/C/30b32d80285d41659daee1731648bf36 is 50, key is test_row_0/C:col10/1732648586793/Put/seqid=0 2024-11-26T19:16:27,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742352_1528 (size=12104) 2024-11-26T19:16:27,722 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/C/30b32d80285d41659daee1731648bf36 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/C/30b32d80285d41659daee1731648bf36 2024-11-26T19:16:27,727 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 66e19d6a9a9885e5b2c786f2e4c45a8e/C of 66e19d6a9a9885e5b2c786f2e4c45a8e into 30b32d80285d41659daee1731648bf36(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T19:16:27,727 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 66e19d6a9a9885e5b2c786f2e4c45a8e: 2024-11-26T19:16:27,727 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., storeName=66e19d6a9a9885e5b2c786f2e4c45a8e/C, priority=13, startTime=1732648587664; duration=0sec 2024-11-26T19:16:27,727 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:16:27,727 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 66e19d6a9a9885e5b2c786f2e4c45a8e:C 2024-11-26T19:16:27,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(8855): Flush requested on 66e19d6a9a9885e5b2c786f2e4c45a8e 2024-11-26T19:16:27,966 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 66e19d6a9a9885e5b2c786f2e4c45a8e 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-26T19:16:27,967 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66e19d6a9a9885e5b2c786f2e4c45a8e, store=A 2024-11-26T19:16:27,967 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:16:27,967 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66e19d6a9a9885e5b2c786f2e4c45a8e, store=B 2024-11-26T19:16:27,967 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:16:27,967 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66e19d6a9a9885e5b2c786f2e4c45a8e, store=C 2024-11-26T19:16:27,967 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:16:27,974 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411263446671321104df8bcb1f58b7f1f962f_66e19d6a9a9885e5b2c786f2e4c45a8e is 50, key is test_row_0/A:col10/1732648587965/Put/seqid=0 2024-11-26T19:16:27,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742353_1529 (size=12154) 2024-11-26T19:16:27,982 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:27,982 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:27,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35456 deadline: 1732648647976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:27,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35426 deadline: 1732648647976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:27,983 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:27,983 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:27,983 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:27,983 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:27,983 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:27,983 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:27,987 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:27,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35454 deadline: 1732648647981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:27,987 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:27,987 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:27,987 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:27,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35416 deadline: 1732648647983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:27,987 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:27,987 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:27,987 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:27,987 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:27,988 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:27,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35438 deadline: 1732648647983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:27,988 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:27,988 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:27,988 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:28,083 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 66e19d6a9a9885e5b2c786f2e4c45a8e#A#compaction#446 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T19:16:28,084 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/A/6f49ed068f664ec4986f5d0d4ed468dd is 175, key is test_row_0/A:col10/1732648586793/Put/seqid=0 2024-11-26T19:16:28,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742354_1530 (size=31058) 2024-11-26T19:16:28,095 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:28,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35456 deadline: 1732648648092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:28,095 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:28,096 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:28,096 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:28,096 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:28,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35426 deadline: 1732648648092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:28,096 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:28,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35454 deadline: 1732648648092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:28,096 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:28,096 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:28,096 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:28,096 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:28,096 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:28,096 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:28,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35416 deadline: 1732648648092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:28,096 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:28,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35438 deadline: 1732648648092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:28,096 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:28,096 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:28,096 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:28,096 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:28,096 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:28,096 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:28,096 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:28,305 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:28,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35456 deadline: 1732648648302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:28,305 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:28,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35426 deadline: 1732648648302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:28,305 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:28,305 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:28,305 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:28,305 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:28,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35454 deadline: 1732648648302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:28,305 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:28,305 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:28,305 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:28,305 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:28,305 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:28,305 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:28,305 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:28,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35416 deadline: 1732648648302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:28,306 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:28,306 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:28,306 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:28,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35438 deadline: 1732648648302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:28,306 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:28,306 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:28,306 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:28,306 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:28,378 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:28,382 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411263446671321104df8bcb1f58b7f1f962f_66e19d6a9a9885e5b2c786f2e4c45a8e to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411263446671321104df8bcb1f58b7f1f962f_66e19d6a9a9885e5b2c786f2e4c45a8e 2024-11-26T19:16:28,382 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/A/a02ced0c15b64db7a036f0c9099f4a51, store: [table=TestAcidGuarantees family=A region=66e19d6a9a9885e5b2c786f2e4c45a8e] 2024-11-26T19:16:28,383 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/A/a02ced0c15b64db7a036f0c9099f4a51 is 175, key is test_row_0/A:col10/1732648587965/Put/seqid=0 2024-11-26T19:16:28,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742355_1531 (size=30955) 2024-11-26T19:16:28,387 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=74, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/A/a02ced0c15b64db7a036f0c9099f4a51 2024-11-26T19:16:28,393 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/B/0bb0305a8ec2489190b2197d1d62d6d2 is 50, key is test_row_0/B:col10/1732648587965/Put/seqid=0 2024-11-26T19:16:28,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742356_1532 (size=12001) 2024-11-26T19:16:28,492 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/A/6f49ed068f664ec4986f5d0d4ed468dd as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/6f49ed068f664ec4986f5d0d4ed468dd 2024-11-26T19:16:28,496 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 66e19d6a9a9885e5b2c786f2e4c45a8e/A of 66e19d6a9a9885e5b2c786f2e4c45a8e into 6f49ed068f664ec4986f5d0d4ed468dd(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T19:16:28,496 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 66e19d6a9a9885e5b2c786f2e4c45a8e: 2024-11-26T19:16:28,496 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., storeName=66e19d6a9a9885e5b2c786f2e4c45a8e/A, priority=13, startTime=1732648587664; duration=0sec 2024-11-26T19:16:28,496 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:16:28,496 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 66e19d6a9a9885e5b2c786f2e4c45a8e:A 2024-11-26T19:16:28,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-26T19:16:28,612 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T19:16:28,613 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T19:16:28,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] procedure2.ProcedureExecutor(1139): Stored pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees 2024-11-26T19:16:28,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-26T19:16:28,614 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T19:16:28,615 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T19:16:28,615 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=148, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T19:16:28,615 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:28,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35456 deadline: 1732648648612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:28,616 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:28,615 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:28,616 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:28,616 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:28,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35426 deadline: 1732648648612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:28,616 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:28,616 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:28,616 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:28,616 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:28,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35454 deadline: 1732648648612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:28,616 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:28,616 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:28,616 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:28,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35438 deadline: 1732648648612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:28,616 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:28,617 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:28,617 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:28,617 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:28,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35416 deadline: 1732648648612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:28,617 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:28,617 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:28,617 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:28,617 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:28,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-26T19:16:28,767 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-26T19:16:28,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e. 2024-11-26T19:16:28,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e. as already flushing 2024-11-26T19:16:28,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e. 2024-11-26T19:16:28,768 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 66e19d6a9a9885e5b2c786f2e4c45a8e, NAME => 'TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:16:28,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 66e19d6a9a9885e5b2c786f2e4c45a8e, NAME => 'TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:16:28,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66e19d6a9a9885e5b2c786f2e4c45a8e, NAME => 'TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66e19d6a9a9885e5b2c786f2e4c45a8e, NAME => 'TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:16:28,797 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=74 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/B/0bb0305a8ec2489190b2197d1d62d6d2 2024-11-26T19:16:28,805 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/C/15c04663c1744d8ca2577de7b569df02 is 50, key is test_row_0/C:col10/1732648587965/Put/seqid=0 2024-11-26T19:16:28,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742357_1533 (size=12001) 2024-11-26T19:16:28,920 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-26T19:16:28,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e. 2024-11-26T19:16:28,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e. as already flushing 2024-11-26T19:16:28,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e. 2024-11-26T19:16:28,921 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 66e19d6a9a9885e5b2c786f2e4c45a8e, NAME => 'TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:16:28,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 66e19d6a9a9885e5b2c786f2e4c45a8e, NAME => 'TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:16:28,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66e19d6a9a9885e5b2c786f2e4c45a8e, NAME => 'TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66e19d6a9a9885e5b2c786f2e4c45a8e, NAME => 'TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:16:28,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-26T19:16:29,073 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-26T19:16:29,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e. 2024-11-26T19:16:29,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e. as already flushing 2024-11-26T19:16:29,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e. 2024-11-26T19:16:29,074 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 66e19d6a9a9885e5b2c786f2e4c45a8e, NAME => 'TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:16:29,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 66e19d6a9a9885e5b2c786f2e4c45a8e, NAME => 'TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:16:29,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66e19d6a9a9885e5b2c786f2e4c45a8e, NAME => 'TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66e19d6a9a9885e5b2c786f2e4c45a8e, NAME => 'TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:16:29,125 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:29,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35456 deadline: 1732648649121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:29,125 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:29,125 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:29,125 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:29,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35454 deadline: 1732648649122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:29,125 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:29,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35426 deadline: 1732648649121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:29,126 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:29,126 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:29,126 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:29,126 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:29,126 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:29,126 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:29,126 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:29,135 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:29,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35438 deadline: 1732648649132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:29,136 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:29,136 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:29,136 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:29,136 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:29,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35416 deadline: 1732648649132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:29,136 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:29,136 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:29,136 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:29,209 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=74 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/C/15c04663c1744d8ca2577de7b569df02 2024-11-26T19:16:29,213 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/A/a02ced0c15b64db7a036f0c9099f4a51 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/a02ced0c15b64db7a036f0c9099f4a51 2024-11-26T19:16:29,216 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/a02ced0c15b64db7a036f0c9099f4a51, entries=150, sequenceid=74, filesize=30.2 K 2024-11-26T19:16:29,217 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/B/0bb0305a8ec2489190b2197d1d62d6d2 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/B/0bb0305a8ec2489190b2197d1d62d6d2 2024-11-26T19:16:29,220 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/B/0bb0305a8ec2489190b2197d1d62d6d2, entries=150, sequenceid=74, filesize=11.7 K 2024-11-26T19:16:29,220 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/C/15c04663c1744d8ca2577de7b569df02 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/C/15c04663c1744d8ca2577de7b569df02 2024-11-26T19:16:29,224 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/C/15c04663c1744d8ca2577de7b569df02, entries=150, sequenceid=74, filesize=11.7 K 2024-11-26T19:16:29,224 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for 66e19d6a9a9885e5b2c786f2e4c45a8e in 1258ms, sequenceid=74, compaction requested=false 2024-11-26T19:16:29,224 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 66e19d6a9a9885e5b2c786f2e4c45a8e: 2024-11-26T19:16:29,225 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-26T19:16:29,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e. 2024-11-26T19:16:29,225 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2902): Flushing 66e19d6a9a9885e5b2c786f2e4c45a8e 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-26T19:16:29,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66e19d6a9a9885e5b2c786f2e4c45a8e, store=A 2024-11-26T19:16:29,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:16:29,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66e19d6a9a9885e5b2c786f2e4c45a8e, store=B 2024-11-26T19:16:29,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:16:29,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66e19d6a9a9885e5b2c786f2e4c45a8e, store=C 2024-11-26T19:16:29,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:16:29,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411260615fc48ba0d4965b996a5c8480cd3f7_66e19d6a9a9885e5b2c786f2e4c45a8e is 50, key is test_row_0/A:col10/1732648587973/Put/seqid=0 2024-11-26T19:16:29,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742358_1534 (size=12154) 2024-11-26T19:16:29,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-26T19:16:29,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:29,639 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411260615fc48ba0d4965b996a5c8480cd3f7_66e19d6a9a9885e5b2c786f2e4c45a8e to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411260615fc48ba0d4965b996a5c8480cd3f7_66e19d6a9a9885e5b2c786f2e4c45a8e 2024-11-26T19:16:29,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/A/c1d35cb2eff54c858e5f6ec86ffdc19e, store: [table=TestAcidGuarantees family=A region=66e19d6a9a9885e5b2c786f2e4c45a8e] 2024-11-26T19:16:29,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/A/c1d35cb2eff54c858e5f6ec86ffdc19e is 175, key is test_row_0/A:col10/1732648587973/Put/seqid=0 2024-11-26T19:16:29,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742359_1535 (size=30955) 2024-11-26T19:16:29,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-26T19:16:30,044 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=88, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/A/c1d35cb2eff54c858e5f6ec86ffdc19e 2024-11-26T19:16:30,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/B/7d89b5162b5b44938a2428c5871d5706 is 50, key is test_row_0/B:col10/1732648587973/Put/seqid=0 2024-11-26T19:16:30,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742360_1536 (size=12001) 2024-11-26T19:16:30,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(8855): Flush requested on 66e19d6a9a9885e5b2c786f2e4c45a8e 2024-11-26T19:16:30,134 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e. as already flushing 2024-11-26T19:16:30,161 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:30,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35426 deadline: 1732648650155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:30,161 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:30,161 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:30,161 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:30,161 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:30,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35438 deadline: 1732648650155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:30,162 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:30,162 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:30,162 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:30,162 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:30,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35416 deadline: 1732648650157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:30,162 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:30,162 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:30,162 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:30,163 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:30,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35456 deadline: 1732648650158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:30,163 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:30,163 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:30,163 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:30,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35454 deadline: 1732648650159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:30,163 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:30,163 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:30,163 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:30,163 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:30,275 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:30,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35426 deadline: 1732648650272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:30,276 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:30,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35438 deadline: 1732648650272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:30,276 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:30,276 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:30,276 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:30,276 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:30,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35416 deadline: 1732648650272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:30,276 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:30,276 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:30,276 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:30,276 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:30,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35456 deadline: 1732648650273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:30,276 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:30,276 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:30,276 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:30,276 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:30,276 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:30,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35454 deadline: 1732648650273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:30,276 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:30,276 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:30,277 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:30,277 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:30,277 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:30,455 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=88 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/B/7d89b5162b5b44938a2428c5871d5706 2024-11-26T19:16:30,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/C/4df303a8950f4039ae03560bb28cbd22 is 50, key is test_row_0/C:col10/1732648587973/Put/seqid=0 2024-11-26T19:16:30,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742361_1537 (size=12001) 2024-11-26T19:16:30,485 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:30,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35426 deadline: 1732648650481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:30,485 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:30,485 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:30,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35438 deadline: 1732648650482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:30,485 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:30,485 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:30,485 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:30,485 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:30,485 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:30,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35416 deadline: 1732648650482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:30,485 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:30,485 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:30,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35456 deadline: 1732648650482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:30,486 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:30,486 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:30,486 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:30,486 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:30,486 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:30,486 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:30,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35454 deadline: 1732648650482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:30,486 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:30,486 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:30,486 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:30,486 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:30,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-26T19:16:30,793 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:30,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35426 deadline: 1732648650791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:30,793 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:30,794 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:30,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35438 deadline: 1732648650792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:30,794 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:30,794 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:30,794 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:30,794 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:30,794 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:30,796 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:30,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35416 deadline: 1732648650792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:30,796 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:30,796 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:30,796 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:30,796 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:30,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35454 deadline: 1732648650792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:30,796 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:30,796 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:30,796 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:30,796 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:30,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35456 deadline: 1732648650792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:30,797 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:30,797 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:30,797 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:30,872 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=88 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/C/4df303a8950f4039ae03560bb28cbd22 2024-11-26T19:16:30,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/A/c1d35cb2eff54c858e5f6ec86ffdc19e as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/c1d35cb2eff54c858e5f6ec86ffdc19e 2024-11-26T19:16:30,880 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/c1d35cb2eff54c858e5f6ec86ffdc19e, entries=150, sequenceid=88, filesize=30.2 K 2024-11-26T19:16:30,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/B/7d89b5162b5b44938a2428c5871d5706 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/B/7d89b5162b5b44938a2428c5871d5706 2024-11-26T19:16:30,883 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/B/7d89b5162b5b44938a2428c5871d5706, entries=150, sequenceid=88, filesize=11.7 K 2024-11-26T19:16:30,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/C/4df303a8950f4039ae03560bb28cbd22 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/C/4df303a8950f4039ae03560bb28cbd22 2024-11-26T19:16:30,887 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/C/4df303a8950f4039ae03560bb28cbd22, entries=150, sequenceid=88, filesize=11.7 K 2024-11-26T19:16:30,888 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 66e19d6a9a9885e5b2c786f2e4c45a8e in 1662ms, sequenceid=88, compaction requested=true 2024-11-26T19:16:30,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2603): Flush status journal for 66e19d6a9a9885e5b2c786f2e4c45a8e: 2024-11-26T19:16:30,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e. 2024-11-26T19:16:30,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=149 2024-11-26T19:16:30,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster(4169): Remote procedure done, pid=149 2024-11-26T19:16:30,891 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=148 2024-11-26T19:16:30,891 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2740 sec 2024-11-26T19:16:30,892 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees in 2.2780 sec 2024-11-26T19:16:31,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(8855): Flush requested on 66e19d6a9a9885e5b2c786f2e4c45a8e 2024-11-26T19:16:31,304 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 66e19d6a9a9885e5b2c786f2e4c45a8e 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-26T19:16:31,306 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66e19d6a9a9885e5b2c786f2e4c45a8e, store=A 2024-11-26T19:16:31,306 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T19:16:31,306 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66e19d6a9a9885e5b2c786f2e4c45a8e, store=B 2024-11-26T19:16:31,306 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T19:16:31,306 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66e19d6a9a9885e5b2c786f2e4c45a8e, store=C 2024-11-26T19:16:31,306 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T19:16:31,314 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411264946c4b3db4a4e0ba9497bde1d48b055_66e19d6a9a9885e5b2c786f2e4c45a8e is 50, key is test_row_0/A:col10/1732648591304/Put/seqid=0 2024-11-26T19:16:31,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742362_1538 (size=14594) 2024-11-26T19:16:31,321 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:31,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35456 deadline: 1732648651314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:31,321 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:31,321 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:31,321 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:31,321 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:31,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35438 deadline: 1732648651315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:31,321 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:31,321 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:31,321 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:31,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35454 deadline: 1732648651316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:31,321 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:31,322 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:31,322 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:31,322 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:31,324 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:31,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35416 deadline: 1732648651317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:31,324 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:31,324 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:31,324 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:31,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35426 deadline: 1732648651319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:31,324 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:31,324 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:31,324 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:31,324 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:31,436 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:31,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35456 deadline: 1732648651431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:31,437 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:31,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35438 deadline: 1732648651431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:31,437 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:31,437 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:31,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35454 deadline: 1732648651432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:31,437 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:31,437 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:31,437 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:31,437 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:31,437 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:31,437 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:31,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35416 deadline: 1732648651432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:31,437 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:31,437 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:31,437 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:31,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35426 deadline: 1732648651432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:31,437 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:31,437 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:31,437 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:31,437 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:31,438 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:31,438 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:31,438 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:31,645 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:31,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35456 deadline: 1732648651642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:31,645 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:31,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35438 deadline: 1732648651642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:31,645 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:31,645 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:31,645 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:31,645 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:31,646 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:31,646 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:31,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35454 deadline: 1732648651642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:31,646 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:31,646 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:31,646 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:31,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35416 deadline: 1732648651642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:31,646 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:31,646 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:31,646 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:31,646 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:31,646 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:31,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35426 deadline: 1732648651642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:31,646 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:31,646 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:31,646 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:31,646 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:31,720 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:31,723 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411264946c4b3db4a4e0ba9497bde1d48b055_66e19d6a9a9885e5b2c786f2e4c45a8e to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411264946c4b3db4a4e0ba9497bde1d48b055_66e19d6a9a9885e5b2c786f2e4c45a8e 2024-11-26T19:16:31,724 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/A/424faf27119a487bae4296921a3dc5f4, store: [table=TestAcidGuarantees family=A region=66e19d6a9a9885e5b2c786f2e4c45a8e] 2024-11-26T19:16:31,725 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/A/424faf27119a487bae4296921a3dc5f4 is 175, key is test_row_0/A:col10/1732648591304/Put/seqid=0 2024-11-26T19:16:31,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742363_1539 (size=39549) 2024-11-26T19:16:31,955 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:31,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35456 deadline: 1732648651951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:31,956 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:31,956 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:31,956 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:31,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35438 deadline: 1732648651951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:31,956 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:31,956 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:31,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35454 deadline: 1732648651952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:31,956 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:31,956 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:31,956 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:31,956 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:31,956 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:31,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35416 deadline: 1732648651952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:31,956 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:31,956 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:31,956 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:31,956 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:31,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35426 deadline: 1732648651952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:31,956 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:31,957 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:31,957 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:31,957 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:31,957 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:32,130 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=109, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/A/424faf27119a487bae4296921a3dc5f4 2024-11-26T19:16:32,137 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/B/9c3cb46b46e244c6bef2e295460b1ab7 is 50, key is test_row_0/B:col10/1732648591304/Put/seqid=0 2024-11-26T19:16:32,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742364_1540 (size=12001) 2024-11-26T19:16:32,466 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:32,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35456 deadline: 1732648652462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:32,466 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:32,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35438 deadline: 1732648652462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:32,467 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:32,467 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:32,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35454 deadline: 1732648652462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:32,467 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:32,467 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:32,467 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:32,467 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:32,467 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:32,467 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:32,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35416 deadline: 1732648652462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:32,467 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:32,467 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:32,467 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:32,467 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:32,467 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:32,467 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:32,467 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:32,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35426 deadline: 1732648652462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:32,467 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:32,467 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:32,467 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:32,541 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=109 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/B/9c3cb46b46e244c6bef2e295460b1ab7 2024-11-26T19:16:32,548 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/C/a1609f64efe4494f9d706ac79e03d3dd is 50, key is test_row_0/C:col10/1732648591304/Put/seqid=0 2024-11-26T19:16:32,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742365_1541 (size=12001) 2024-11-26T19:16:32,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-26T19:16:32,772 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T19:16:32,772 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T19:16:32,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] procedure2.ProcedureExecutor(1139): Stored pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees 2024-11-26T19:16:32,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-26T19:16:32,774 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T19:16:32,775 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T19:16:32,775 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T19:16:32,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-26T19:16:32,927 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-26T19:16:32,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e. 2024-11-26T19:16:32,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e. as already flushing 2024-11-26T19:16:32,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e. 2024-11-26T19:16:32,927 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 66e19d6a9a9885e5b2c786f2e4c45a8e, NAME => 'TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:16:32,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 66e19d6a9a9885e5b2c786f2e4c45a8e, NAME => 'TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:16:32,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66e19d6a9a9885e5b2c786f2e4c45a8e, NAME => 'TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66e19d6a9a9885e5b2c786f2e4c45a8e, NAME => 'TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:16:32,951 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=109 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/C/a1609f64efe4494f9d706ac79e03d3dd 2024-11-26T19:16:32,955 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/A/424faf27119a487bae4296921a3dc5f4 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/424faf27119a487bae4296921a3dc5f4 2024-11-26T19:16:32,958 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/424faf27119a487bae4296921a3dc5f4, entries=200, sequenceid=109, filesize=38.6 K 2024-11-26T19:16:32,959 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/B/9c3cb46b46e244c6bef2e295460b1ab7 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/B/9c3cb46b46e244c6bef2e295460b1ab7 2024-11-26T19:16:32,962 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/B/9c3cb46b46e244c6bef2e295460b1ab7, entries=150, sequenceid=109, filesize=11.7 K 2024-11-26T19:16:32,962 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/C/a1609f64efe4494f9d706ac79e03d3dd as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/C/a1609f64efe4494f9d706ac79e03d3dd 2024-11-26T19:16:32,965 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/C/a1609f64efe4494f9d706ac79e03d3dd, entries=150, sequenceid=109, filesize=11.7 K 2024-11-26T19:16:32,966 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=60.38 KB/61830 for 66e19d6a9a9885e5b2c786f2e4c45a8e in 1662ms, sequenceid=109, compaction requested=true 2024-11-26T19:16:32,966 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 66e19d6a9a9885e5b2c786f2e4c45a8e: 2024-11-26T19:16:32,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 66e19d6a9a9885e5b2c786f2e4c45a8e:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T19:16:32,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:16:32,966 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T19:16:32,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 66e19d6a9a9885e5b2c786f2e4c45a8e:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T19:16:32,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:16:32,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 66e19d6a9a9885e5b2c786f2e4c45a8e:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T19:16:32,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T19:16:32,966 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T19:16:32,967 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 132517 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T19:16:32,967 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T19:16:32,967 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1541): 66e19d6a9a9885e5b2c786f2e4c45a8e/A is initiating minor compaction (all files) 2024-11-26T19:16:32,967 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1541): 66e19d6a9a9885e5b2c786f2e4c45a8e/B is initiating minor compaction (all files) 2024-11-26T19:16:32,967 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 66e19d6a9a9885e5b2c786f2e4c45a8e/B in TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e. 2024-11-26T19:16:32,967 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 66e19d6a9a9885e5b2c786f2e4c45a8e/A in TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e. 2024-11-26T19:16:32,967 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/6f49ed068f664ec4986f5d0d4ed468dd, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/a02ced0c15b64db7a036f0c9099f4a51, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/c1d35cb2eff54c858e5f6ec86ffdc19e, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/424faf27119a487bae4296921a3dc5f4] into tmpdir=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp, totalSize=129.4 K 2024-11-26T19:16:32,967 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/B/a2569dff74ba4043b514760294e8b147, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/B/0bb0305a8ec2489190b2197d1d62d6d2, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/B/7d89b5162b5b44938a2428c5871d5706, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/B/9c3cb46b46e244c6bef2e295460b1ab7] into tmpdir=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp, totalSize=47.0 K 2024-11-26T19:16:32,967 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e. 2024-11-26T19:16:32,968 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e. files: [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/6f49ed068f664ec4986f5d0d4ed468dd, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/a02ced0c15b64db7a036f0c9099f4a51, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/c1d35cb2eff54c858e5f6ec86ffdc19e, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/424faf27119a487bae4296921a3dc5f4] 2024-11-26T19:16:32,968 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting a2569dff74ba4043b514760294e8b147, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732648586683 2024-11-26T19:16:32,968 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6f49ed068f664ec4986f5d0d4ed468dd, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732648586683 2024-11-26T19:16:32,968 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting 0bb0305a8ec2489190b2197d1d62d6d2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1732648587964 2024-11-26T19:16:32,968 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting a02ced0c15b64db7a036f0c9099f4a51, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1732648587964 2024-11-26T19:16:32,968 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting 7d89b5162b5b44938a2428c5871d5706, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1732648587973 2024-11-26T19:16:32,969 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting c1d35cb2eff54c858e5f6ec86ffdc19e, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1732648587973 2024-11-26T19:16:32,969 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting 9c3cb46b46e244c6bef2e295460b1ab7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1732648590157 2024-11-26T19:16:32,969 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting 424faf27119a487bae4296921a3dc5f4, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1732648590147 2024-11-26T19:16:32,975 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 66e19d6a9a9885e5b2c786f2e4c45a8e#B#compaction#458 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T19:16:32,976 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/B/6100ea3d0fe04142b1e874fbcc533cf3 is 50, key is test_row_0/B:col10/1732648591304/Put/seqid=0 2024-11-26T19:16:32,977 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=66e19d6a9a9885e5b2c786f2e4c45a8e] 2024-11-26T19:16:32,979 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112681cd04d9f6184418b0b01d57b7c8f33c_66e19d6a9a9885e5b2c786f2e4c45a8e store=[table=TestAcidGuarantees family=A region=66e19d6a9a9885e5b2c786f2e4c45a8e] 2024-11-26T19:16:32,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742366_1542 (size=12241) 2024-11-26T19:16:32,982 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112681cd04d9f6184418b0b01d57b7c8f33c_66e19d6a9a9885e5b2c786f2e4c45a8e, store=[table=TestAcidGuarantees family=A region=66e19d6a9a9885e5b2c786f2e4c45a8e] 2024-11-26T19:16:32,982 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112681cd04d9f6184418b0b01d57b7c8f33c_66e19d6a9a9885e5b2c786f2e4c45a8e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=66e19d6a9a9885e5b2c786f2e4c45a8e] 2024-11-26T19:16:32,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742367_1543 (size=4469) 2024-11-26T19:16:32,986 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/B/6100ea3d0fe04142b1e874fbcc533cf3 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/B/6100ea3d0fe04142b1e874fbcc533cf3 2024-11-26T19:16:32,986 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 66e19d6a9a9885e5b2c786f2e4c45a8e#A#compaction#459 average throughput is 2.71 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T19:16:32,987 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/A/fc89df7a6f5c497bb05ae88db2ac6aeb is 175, key is test_row_0/A:col10/1732648591304/Put/seqid=0 2024-11-26T19:16:32,990 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 66e19d6a9a9885e5b2c786f2e4c45a8e/B of 66e19d6a9a9885e5b2c786f2e4c45a8e into 6100ea3d0fe04142b1e874fbcc533cf3(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T19:16:32,990 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 66e19d6a9a9885e5b2c786f2e4c45a8e: 2024-11-26T19:16:32,990 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., storeName=66e19d6a9a9885e5b2c786f2e4c45a8e/B, priority=12, startTime=1732648592966; duration=0sec 2024-11-26T19:16:32,990 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T19:16:32,990 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 66e19d6a9a9885e5b2c786f2e4c45a8e:B 2024-11-26T19:16:32,991 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T19:16:32,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742368_1544 (size=31195) 2024-11-26T19:16:32,992 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T19:16:32,992 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1541): 66e19d6a9a9885e5b2c786f2e4c45a8e/C is initiating minor compaction (all files) 2024-11-26T19:16:32,992 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 66e19d6a9a9885e5b2c786f2e4c45a8e/C in TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e. 2024-11-26T19:16:32,992 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/C/30b32d80285d41659daee1731648bf36, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/C/15c04663c1744d8ca2577de7b569df02, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/C/4df303a8950f4039ae03560bb28cbd22, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/C/a1609f64efe4494f9d706ac79e03d3dd] into tmpdir=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp, totalSize=47.0 K 2024-11-26T19:16:32,993 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting 30b32d80285d41659daee1731648bf36, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732648586683 2024-11-26T19:16:32,993 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting 15c04663c1744d8ca2577de7b569df02, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1732648587964 2024-11-26T19:16:32,994 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting 4df303a8950f4039ae03560bb28cbd22, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1732648587973 2024-11-26T19:16:32,994 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting a1609f64efe4494f9d706ac79e03d3dd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1732648590157 2024-11-26T19:16:32,995 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/A/fc89df7a6f5c497bb05ae88db2ac6aeb as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/fc89df7a6f5c497bb05ae88db2ac6aeb 2024-11-26T19:16:32,999 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 66e19d6a9a9885e5b2c786f2e4c45a8e/A of 66e19d6a9a9885e5b2c786f2e4c45a8e into fc89df7a6f5c497bb05ae88db2ac6aeb(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T19:16:32,999 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 66e19d6a9a9885e5b2c786f2e4c45a8e: 2024-11-26T19:16:33,000 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., storeName=66e19d6a9a9885e5b2c786f2e4c45a8e/A, priority=12, startTime=1732648592966; duration=0sec 2024-11-26T19:16:33,000 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:16:33,000 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 66e19d6a9a9885e5b2c786f2e4c45a8e:A 2024-11-26T19:16:33,002 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 66e19d6a9a9885e5b2c786f2e4c45a8e#C#compaction#460 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T19:16:33,002 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/C/b519d0bf2c1744aab5fc25030aa9a062 is 50, key is test_row_0/C:col10/1732648591304/Put/seqid=0 2024-11-26T19:16:33,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742369_1545 (size=12241) 2024-11-26T19:16:33,009 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/C/b519d0bf2c1744aab5fc25030aa9a062 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/C/b519d0bf2c1744aab5fc25030aa9a062 2024-11-26T19:16:33,013 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 66e19d6a9a9885e5b2c786f2e4c45a8e/C of 66e19d6a9a9885e5b2c786f2e4c45a8e into b519d0bf2c1744aab5fc25030aa9a062(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T19:16:33,013 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 66e19d6a9a9885e5b2c786f2e4c45a8e: 2024-11-26T19:16:33,013 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., storeName=66e19d6a9a9885e5b2c786f2e4c45a8e/C, priority=12, startTime=1732648592966; duration=0sec 2024-11-26T19:16:33,013 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:16:33,013 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 66e19d6a9a9885e5b2c786f2e4c45a8e:C 2024-11-26T19:16:33,080 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-26T19:16:33,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e. 2024-11-26T19:16:33,080 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2902): Flushing 66e19d6a9a9885e5b2c786f2e4c45a8e 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-26T19:16:33,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66e19d6a9a9885e5b2c786f2e4c45a8e, store=A 2024-11-26T19:16:33,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:16:33,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66e19d6a9a9885e5b2c786f2e4c45a8e, store=B 2024-11-26T19:16:33,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:16:33,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66e19d6a9a9885e5b2c786f2e4c45a8e, store=C 2024-11-26T19:16:33,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:16:33,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112653d591040fd84deb93623e97b63da51f_66e19d6a9a9885e5b2c786f2e4c45a8e is 50, key is test_row_0/A:col10/1732648591316/Put/seqid=0 2024-11-26T19:16:33,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742370_1546 (size=12154) 2024-11-26T19:16:33,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:33,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-26T19:16:33,094 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112653d591040fd84deb93623e97b63da51f_66e19d6a9a9885e5b2c786f2e4c45a8e to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112653d591040fd84deb93623e97b63da51f_66e19d6a9a9885e5b2c786f2e4c45a8e 2024-11-26T19:16:33,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/A/549a987d77de492d8546f7ef8da171db, store: [table=TestAcidGuarantees family=A region=66e19d6a9a9885e5b2c786f2e4c45a8e] 2024-11-26T19:16:33,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/A/549a987d77de492d8546f7ef8da171db is 175, key is test_row_0/A:col10/1732648591316/Put/seqid=0 2024-11-26T19:16:33,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742371_1547 (size=30955) 2024-11-26T19:16:33,106 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=124, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/A/549a987d77de492d8546f7ef8da171db 2024-11-26T19:16:33,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/B/53e57dc86ac544319bee86011f6ff6ca is 50, key is test_row_0/B:col10/1732648591316/Put/seqid=0 2024-11-26T19:16:33,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742372_1548 (size=12001) 2024-11-26T19:16:33,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-26T19:16:33,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(8855): Flush requested on 66e19d6a9a9885e5b2c786f2e4c45a8e 2024-11-26T19:16:33,474 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e. as already flushing 2024-11-26T19:16:33,499 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:33,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35416 deadline: 1732648653494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:33,500 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:33,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35426 deadline: 1732648653494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:33,500 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:33,500 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:33,500 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:33,500 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:33,500 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:33,500 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:33,500 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:33,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35456 deadline: 1732648653495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:33,500 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:33,500 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:33,500 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:33,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35454 deadline: 1732648653496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:33,500 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:33,501 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:33,501 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:33,501 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:33,503 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:33,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35438 deadline: 1732648653499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:33,503 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:33,503 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:33,504 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:33,517 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=124 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/B/53e57dc86ac544319bee86011f6ff6ca 2024-11-26T19:16:33,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/C/569f54ee3b00493da947546d9599b245 is 50, key is test_row_0/C:col10/1732648591316/Put/seqid=0 2024-11-26T19:16:33,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742373_1549 (size=12001) 2024-11-26T19:16:33,604 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:33,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35416 deadline: 1732648653601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:33,604 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:33,604 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:33,604 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:33,615 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:33,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35426 deadline: 1732648653612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:33,615 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:33,615 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:33,615 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:33,615 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:33,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35456 deadline: 1732648653612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:33,616 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:33,616 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:33,616 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:33,616 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:33,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35454 deadline: 1732648653612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:33,616 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:33,616 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:33,616 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:33,616 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:33,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35438 deadline: 1732648653612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:33,616 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:33,616 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:33,616 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:33,814 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:33,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35416 deadline: 1732648653812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:33,815 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:33,815 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:33,815 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:33,824 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:33,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35426 deadline: 1732648653822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:33,824 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:33,824 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:33,824 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:33,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35456 deadline: 1732648653822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:33,824 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:33,824 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:33,824 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:33,824 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:33,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35454 deadline: 1732648653822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:33,824 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:33,825 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:33,825 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:33,825 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:33,826 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:33,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35438 deadline: 1732648653822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:33,826 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:33,826 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:33,826 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:33,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-26T19:16:33,927 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=124 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/C/569f54ee3b00493da947546d9599b245 2024-11-26T19:16:33,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/A/549a987d77de492d8546f7ef8da171db as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/549a987d77de492d8546f7ef8da171db 2024-11-26T19:16:33,934 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/549a987d77de492d8546f7ef8da171db, entries=150, sequenceid=124, filesize=30.2 K 2024-11-26T19:16:33,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/B/53e57dc86ac544319bee86011f6ff6ca as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/B/53e57dc86ac544319bee86011f6ff6ca 2024-11-26T19:16:33,938 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/B/53e57dc86ac544319bee86011f6ff6ca, entries=150, sequenceid=124, filesize=11.7 K 2024-11-26T19:16:33,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/C/569f54ee3b00493da947546d9599b245 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/C/569f54ee3b00493da947546d9599b245 2024-11-26T19:16:33,942 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/C/569f54ee3b00493da947546d9599b245, entries=150, sequenceid=124, filesize=11.7 K 2024-11-26T19:16:33,943 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 66e19d6a9a9885e5b2c786f2e4c45a8e in 863ms, sequenceid=124, compaction requested=false 2024-11-26T19:16:33,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2603): Flush status journal for 66e19d6a9a9885e5b2c786f2e4c45a8e: 2024-11-26T19:16:33,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e. 2024-11-26T19:16:33,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=151 2024-11-26T19:16:33,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster(4169): Remote procedure done, pid=151 2024-11-26T19:16:33,947 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=150 2024-11-26T19:16:33,947 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1700 sec 2024-11-26T19:16:33,949 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees in 1.1760 sec 2024-11-26T19:16:34,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(8855): Flush requested on 66e19d6a9a9885e5b2c786f2e4c45a8e 2024-11-26T19:16:34,125 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 66e19d6a9a9885e5b2c786f2e4c45a8e 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-26T19:16:34,126 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66e19d6a9a9885e5b2c786f2e4c45a8e, store=A 2024-11-26T19:16:34,126 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T19:16:34,126 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66e19d6a9a9885e5b2c786f2e4c45a8e, store=B 2024-11-26T19:16:34,126 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T19:16:34,126 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66e19d6a9a9885e5b2c786f2e4c45a8e, store=C 2024-11-26T19:16:34,126 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T19:16:34,135 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411269251111d12fb4f379dddb3a2caa08920_66e19d6a9a9885e5b2c786f2e4c45a8e is 50, key is test_row_0/A:col10/1732648593492/Put/seqid=0 2024-11-26T19:16:34,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742374_1550 (size=12304) 2024-11-26T19:16:34,145 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:34,146 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:34,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35438 deadline: 1732648654139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:34,147 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:34,147 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:34,147 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:34,147 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:34,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35454 deadline: 1732648654142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:34,147 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:34,147 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:34,147 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:34,147 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:34,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35416 deadline: 1732648654144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:34,147 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:34,148 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:34,148 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:34,148 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:34,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35456 deadline: 1732648654145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:34,149 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:34,149 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:34,149 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:34,150 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411269251111d12fb4f379dddb3a2caa08920_66e19d6a9a9885e5b2c786f2e4c45a8e to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411269251111d12fb4f379dddb3a2caa08920_66e19d6a9a9885e5b2c786f2e4c45a8e 2024-11-26T19:16:34,150 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/A/52297031376c4960b8d295fc2485d9c0, store: [table=TestAcidGuarantees family=A region=66e19d6a9a9885e5b2c786f2e4c45a8e] 2024-11-26T19:16:34,151 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/A/52297031376c4960b8d295fc2485d9c0 is 175, key is test_row_0/A:col10/1732648593492/Put/seqid=0 2024-11-26T19:16:34,151 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:34,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35426 deadline: 1732648654147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:34,151 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:34,151 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:34,151 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:34,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742375_1551 (size=31105) 2024-11-26T19:16:34,253 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:34,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35438 deadline: 1732648654252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:34,254 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:34,254 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:34,254 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:34,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35454 deadline: 1732648654252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:34,254 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:34,254 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:34,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35416 deadline: 1732648654252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:34,254 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:34,254 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:34,254 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:34,254 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:34,254 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:34,254 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:34,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35456 deadline: 1732648654253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:34,255 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:34,255 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:34,255 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:34,255 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:34,265 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:34,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35426 deadline: 1732648654262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:34,266 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:34,266 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:34,266 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:34,466 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:34,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35454 deadline: 1732648654462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:34,466 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:34,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35456 deadline: 1732648654462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:34,466 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:34,466 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:34,467 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:34,467 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:34,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35438 deadline: 1732648654462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:34,467 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:34,467 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:34,467 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:34,467 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:34,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35416 deadline: 1732648654462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:34,467 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:34,467 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:34,467 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:34,467 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:34,467 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:34,467 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:34,472 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:34,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35426 deadline: 1732648654472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:34,473 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:34,473 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:34,473 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:34,555 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=145, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/A/52297031376c4960b8d295fc2485d9c0 2024-11-26T19:16:34,563 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/B/95f3855b9b474384baaeea19ab4290c0 is 50, key is test_row_0/B:col10/1732648593492/Put/seqid=0 2024-11-26T19:16:34,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742376_1552 (size=12151) 2024-11-26T19:16:34,567 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/B/95f3855b9b474384baaeea19ab4290c0 2024-11-26T19:16:34,574 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/C/3653f949eea7465cb4954db45b016357 is 50, key is test_row_0/C:col10/1732648593492/Put/seqid=0 2024-11-26T19:16:34,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742377_1553 (size=12151) 2024-11-26T19:16:34,775 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:34,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35454 deadline: 1732648654771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:34,775 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:34,776 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:34,776 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:34,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35456 deadline: 1732648654772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:34,776 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:34,776 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:34,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35438 deadline: 1732648654772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:34,776 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:34,776 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:34,776 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:34,776 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:34,776 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:34,776 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:34,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35416 deadline: 1732648654772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:34,776 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:34,776 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:34,776 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:34,777 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:34,786 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:34,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35426 deadline: 1732648654782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:34,786 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:34,786 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:34,786 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:34,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-26T19:16:34,922 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T19:16:34,923 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T19:16:34,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] procedure2.ProcedureExecutor(1139): Stored pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees 2024-11-26T19:16:34,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-26T19:16:34,924 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T19:16:34,925 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T19:16:34,925 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T19:16:34,977 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/C/3653f949eea7465cb4954db45b016357 2024-11-26T19:16:34,981 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/A/52297031376c4960b8d295fc2485d9c0 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/52297031376c4960b8d295fc2485d9c0 2024-11-26T19:16:34,984 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/52297031376c4960b8d295fc2485d9c0, entries=150, sequenceid=145, filesize=30.4 K 2024-11-26T19:16:34,984 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/B/95f3855b9b474384baaeea19ab4290c0 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/B/95f3855b9b474384baaeea19ab4290c0 2024-11-26T19:16:34,987 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/B/95f3855b9b474384baaeea19ab4290c0, entries=150, sequenceid=145, filesize=11.9 K 2024-11-26T19:16:34,992 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/C/3653f949eea7465cb4954db45b016357 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/C/3653f949eea7465cb4954db45b016357 2024-11-26T19:16:34,996 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/C/3653f949eea7465cb4954db45b016357, entries=150, sequenceid=145, filesize=11.9 K 2024-11-26T19:16:34,997 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for 66e19d6a9a9885e5b2c786f2e4c45a8e in 872ms, sequenceid=145, compaction requested=true 2024-11-26T19:16:34,997 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 66e19d6a9a9885e5b2c786f2e4c45a8e: 2024-11-26T19:16:34,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 66e19d6a9a9885e5b2c786f2e4c45a8e:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T19:16:34,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:16:34,997 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T19:16:34,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 66e19d6a9a9885e5b2c786f2e4c45a8e:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T19:16:34,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:16:34,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 66e19d6a9a9885e5b2c786f2e4c45a8e:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T19:16:34,997 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T19:16:34,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T19:16:34,998 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T19:16:34,998 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93255 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T19:16:34,998 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1541): 66e19d6a9a9885e5b2c786f2e4c45a8e/B is initiating minor compaction (all files) 2024-11-26T19:16:34,998 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1541): 66e19d6a9a9885e5b2c786f2e4c45a8e/A is initiating minor compaction (all files) 2024-11-26T19:16:34,998 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 66e19d6a9a9885e5b2c786f2e4c45a8e/B in TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e. 2024-11-26T19:16:34,998 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 66e19d6a9a9885e5b2c786f2e4c45a8e/A in TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e. 2024-11-26T19:16:34,998 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/B/6100ea3d0fe04142b1e874fbcc533cf3, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/B/53e57dc86ac544319bee86011f6ff6ca, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/B/95f3855b9b474384baaeea19ab4290c0] into tmpdir=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp, totalSize=35.5 K 2024-11-26T19:16:34,998 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/fc89df7a6f5c497bb05ae88db2ac6aeb, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/549a987d77de492d8546f7ef8da171db, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/52297031376c4960b8d295fc2485d9c0] into tmpdir=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp, totalSize=91.1 K 2024-11-26T19:16:34,998 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e. 2024-11-26T19:16:34,998 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e. files: [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/fc89df7a6f5c497bb05ae88db2ac6aeb, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/549a987d77de492d8546f7ef8da171db, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/52297031376c4960b8d295fc2485d9c0] 2024-11-26T19:16:34,998 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting fc89df7a6f5c497bb05ae88db2ac6aeb, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1732648590157 2024-11-26T19:16:34,998 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting 6100ea3d0fe04142b1e874fbcc533cf3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1732648590157 2024-11-26T19:16:34,999 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting 549a987d77de492d8546f7ef8da171db, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=124, earliestPutTs=1732648591313 2024-11-26T19:16:34,999 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting 53e57dc86ac544319bee86011f6ff6ca, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=124, earliestPutTs=1732648591313 2024-11-26T19:16:34,999 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting 52297031376c4960b8d295fc2485d9c0, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1732648593492 2024-11-26T19:16:34,999 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting 95f3855b9b474384baaeea19ab4290c0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1732648593492 2024-11-26T19:16:35,008 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=66e19d6a9a9885e5b2c786f2e4c45a8e] 2024-11-26T19:16:35,008 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 66e19d6a9a9885e5b2c786f2e4c45a8e#B#compaction#467 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T19:16:35,009 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/B/cd980fc48f114e9ea2f096da0d4a4d09 is 50, key is test_row_0/B:col10/1732648593492/Put/seqid=0 2024-11-26T19:16:35,010 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241126decb5b0494194543863d8cb3bdc8ac92_66e19d6a9a9885e5b2c786f2e4c45a8e store=[table=TestAcidGuarantees family=A region=66e19d6a9a9885e5b2c786f2e4c45a8e] 2024-11-26T19:16:35,011 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241126decb5b0494194543863d8cb3bdc8ac92_66e19d6a9a9885e5b2c786f2e4c45a8e, store=[table=TestAcidGuarantees family=A region=66e19d6a9a9885e5b2c786f2e4c45a8e] 2024-11-26T19:16:35,011 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126decb5b0494194543863d8cb3bdc8ac92_66e19d6a9a9885e5b2c786f2e4c45a8e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=66e19d6a9a9885e5b2c786f2e4c45a8e] 2024-11-26T19:16:35,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742378_1554 (size=12493) 2024-11-26T19:16:35,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742379_1555 (size=4469) 2024-11-26T19:16:35,020 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 66e19d6a9a9885e5b2c786f2e4c45a8e#A#compaction#468 average throughput is 2.04 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T19:16:35,020 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/A/bb50fc14458e4ff493f25015fe5a5e0f is 175, key is test_row_0/A:col10/1732648593492/Put/seqid=0 2024-11-26T19:16:35,022 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/B/cd980fc48f114e9ea2f096da0d4a4d09 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/B/cd980fc48f114e9ea2f096da0d4a4d09 2024-11-26T19:16:35,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742380_1556 (size=31447) 2024-11-26T19:16:35,027 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 66e19d6a9a9885e5b2c786f2e4c45a8e/B of 66e19d6a9a9885e5b2c786f2e4c45a8e into cd980fc48f114e9ea2f096da0d4a4d09(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T19:16:35,027 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 66e19d6a9a9885e5b2c786f2e4c45a8e: 2024-11-26T19:16:35,027 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., storeName=66e19d6a9a9885e5b2c786f2e4c45a8e/B, priority=13, startTime=1732648594997; duration=0sec 2024-11-26T19:16:35,027 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T19:16:35,028 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 66e19d6a9a9885e5b2c786f2e4c45a8e:B 2024-11-26T19:16:35,028 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T19:16:35,029 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T19:16:35,029 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1541): 66e19d6a9a9885e5b2c786f2e4c45a8e/C is initiating minor compaction (all files) 2024-11-26T19:16:35,029 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 66e19d6a9a9885e5b2c786f2e4c45a8e/C in TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e. 2024-11-26T19:16:35,029 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/C/b519d0bf2c1744aab5fc25030aa9a062, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/C/569f54ee3b00493da947546d9599b245, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/C/3653f949eea7465cb4954db45b016357] into tmpdir=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp, totalSize=35.5 K 2024-11-26T19:16:35,029 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting b519d0bf2c1744aab5fc25030aa9a062, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1732648590157 2024-11-26T19:16:35,030 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting 569f54ee3b00493da947546d9599b245, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=124, earliestPutTs=1732648591313 2024-11-26T19:16:35,030 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting 3653f949eea7465cb4954db45b016357, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1732648593492 2024-11-26T19:16:35,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-26T19:16:35,038 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 66e19d6a9a9885e5b2c786f2e4c45a8e#C#compaction#469 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T19:16:35,038 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/C/c99db25f98c64c4a878c80e1720d6611 is 50, key is test_row_0/C:col10/1732648593492/Put/seqid=0 2024-11-26T19:16:35,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742381_1557 (size=12493) 2024-11-26T19:16:35,078 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-26T19:16:35,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e. 2024-11-26T19:16:35,078 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2902): Flushing 66e19d6a9a9885e5b2c786f2e4c45a8e 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-26T19:16:35,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66e19d6a9a9885e5b2c786f2e4c45a8e, store=A 2024-11-26T19:16:35,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:16:35,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66e19d6a9a9885e5b2c786f2e4c45a8e, store=B 2024-11-26T19:16:35,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:16:35,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66e19d6a9a9885e5b2c786f2e4c45a8e, store=C 2024-11-26T19:16:35,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:16:35,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126c7566238493045509a38f9b11a367c2b_66e19d6a9a9885e5b2c786f2e4c45a8e is 50, key is test_row_0/A:col10/1732648594145/Put/seqid=0 2024-11-26T19:16:35,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742382_1558 (size=12304) 2024-11-26T19:16:35,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-26T19:16:35,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(8855): Flush requested on 66e19d6a9a9885e5b2c786f2e4c45a8e 2024-11-26T19:16:35,284 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e. as already flushing 2024-11-26T19:16:35,309 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:35,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35438 deadline: 1732648655303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:35,309 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:35,309 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:35,309 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:35,311 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732648595301, max timestamp=1732648595308Num uniques 859; 2024-11-26T19:16:35,311 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-26T19:16:35,311 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732648595301, max timestamp=1732648595308Num uniques 859; 2024-11-26T19:16:35,312 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-26T19:16:35,312 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732648595301, max timestamp=1732648595308Num uniques 859; 2024-11-26T19:16:35,312 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-26T19:16:35,338 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:35,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35456 deadline: 1732648655331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:35,338 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:35,338 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:35,338 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:35,343 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:35,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35416 deadline: 1732648655335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:35,343 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:35,343 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:35,343 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:35,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35426 deadline: 1732648655337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:35,343 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:35,343 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:35,343 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:35,343 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:35,344 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:35,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35454 deadline: 1732648655338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:35,344 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:35,344 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:35,344 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:35,416 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:35,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35438 deadline: 1732648655411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:35,416 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:35,416 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:35,416 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:35,430 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/A/bb50fc14458e4ff493f25015fe5a5e0f as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/bb50fc14458e4ff493f25015fe5a5e0f 2024-11-26T19:16:35,434 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 66e19d6a9a9885e5b2c786f2e4c45a8e/A of 66e19d6a9a9885e5b2c786f2e4c45a8e into bb50fc14458e4ff493f25015fe5a5e0f(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T19:16:35,434 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 66e19d6a9a9885e5b2c786f2e4c45a8e: 2024-11-26T19:16:35,434 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., storeName=66e19d6a9a9885e5b2c786f2e4c45a8e/A, priority=13, startTime=1732648594997; duration=0sec 2024-11-26T19:16:35,434 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:16:35,434 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 66e19d6a9a9885e5b2c786f2e4c45a8e:A 2024-11-26T19:16:35,443 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:35,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35456 deadline: 1732648655442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:35,444 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:35,444 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:35,444 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:35,446 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/C/c99db25f98c64c4a878c80e1720d6611 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/C/c99db25f98c64c4a878c80e1720d6611 2024-11-26T19:16:35,450 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 66e19d6a9a9885e5b2c786f2e4c45a8e/C of 66e19d6a9a9885e5b2c786f2e4c45a8e into c99db25f98c64c4a878c80e1720d6611(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T19:16:35,451 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 66e19d6a9a9885e5b2c786f2e4c45a8e: 2024-11-26T19:16:35,451 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., storeName=66e19d6a9a9885e5b2c786f2e4c45a8e/C, priority=13, startTime=1732648594997; duration=0sec 2024-11-26T19:16:35,451 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:16:35,451 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 66e19d6a9a9885e5b2c786f2e4c45a8e:C 2024-11-26T19:16:35,455 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:35,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35416 deadline: 1732648655451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:35,455 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:35,455 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:35,455 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:35,455 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:35,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35426 deadline: 1732648655451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:35,456 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:35,456 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:35,456 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:35,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35454 deadline: 1732648655451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:35,456 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:35,456 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:35,456 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:35,456 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:35,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:35,500 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126c7566238493045509a38f9b11a367c2b_66e19d6a9a9885e5b2c786f2e4c45a8e to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126c7566238493045509a38f9b11a367c2b_66e19d6a9a9885e5b2c786f2e4c45a8e 2024-11-26T19:16:35,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/A/d9629b526b2840e29e5f097c0c495118, store: [table=TestAcidGuarantees family=A region=66e19d6a9a9885e5b2c786f2e4c45a8e] 2024-11-26T19:16:35,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/A/d9629b526b2840e29e5f097c0c495118 is 175, key is test_row_0/A:col10/1732648594145/Put/seqid=0 2024-11-26T19:16:35,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742383_1559 (size=31105) 2024-11-26T19:16:35,506 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=157, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/A/d9629b526b2840e29e5f097c0c495118 2024-11-26T19:16:35,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/B/db4363f29ea240e9920f9452c1c0b996 is 50, key is test_row_0/B:col10/1732648594145/Put/seqid=0 2024-11-26T19:16:35,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742384_1560 (size=12151) 2024-11-26T19:16:35,515 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/B/db4363f29ea240e9920f9452c1c0b996 2024-11-26T19:16:35,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/C/2de53e2a3cd84482b3e52ec82eda114f is 50, key is test_row_0/C:col10/1732648594145/Put/seqid=0 2024-11-26T19:16:35,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742385_1561 (size=12151) 2024-11-26T19:16:35,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-26T19:16:35,624 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:35,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35438 deadline: 1732648655621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:35,625 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:35,625 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:35,625 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:35,656 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:35,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35456 deadline: 1732648655652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:35,657 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:35,657 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:35,657 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:35,665 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:35,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35416 deadline: 1732648655662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:35,665 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:35,665 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:35,666 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:35,666 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:35,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35426 deadline: 1732648655662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:35,666 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:35,666 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:35,666 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:35,666 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:35,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35454 deadline: 1732648655662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:35,666 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:35,666 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:35,666 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:35,925 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/C/2de53e2a3cd84482b3e52ec82eda114f 2024-11-26T19:16:35,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/A/d9629b526b2840e29e5f097c0c495118 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/d9629b526b2840e29e5f097c0c495118 2024-11-26T19:16:35,932 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/d9629b526b2840e29e5f097c0c495118, entries=150, sequenceid=157, filesize=30.4 K 2024-11-26T19:16:35,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/B/db4363f29ea240e9920f9452c1c0b996 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/B/db4363f29ea240e9920f9452c1c0b996 2024-11-26T19:16:35,936 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/B/db4363f29ea240e9920f9452c1c0b996, entries=150, sequenceid=157, filesize=11.9 K 2024-11-26T19:16:35,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/C/2de53e2a3cd84482b3e52ec82eda114f as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/C/2de53e2a3cd84482b3e52ec82eda114f 2024-11-26T19:16:35,937 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:35,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35438 deadline: 1732648655932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:35,938 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:35,938 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:35,938 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:35,939 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/C/2de53e2a3cd84482b3e52ec82eda114f, entries=150, sequenceid=157, filesize=11.9 K 2024-11-26T19:16:35,940 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 66e19d6a9a9885e5b2c786f2e4c45a8e in 862ms, sequenceid=157, compaction requested=false 2024-11-26T19:16:35,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2603): Flush status journal for 66e19d6a9a9885e5b2c786f2e4c45a8e: 2024-11-26T19:16:35,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e. 2024-11-26T19:16:35,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=153 2024-11-26T19:16:35,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster(4169): Remote procedure done, pid=153 2024-11-26T19:16:35,943 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-11-26T19:16:35,943 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0170 sec 2024-11-26T19:16:35,945 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees in 1.0210 sec 2024-11-26T19:16:35,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(8855): Flush requested on 66e19d6a9a9885e5b2c786f2e4c45a8e 2024-11-26T19:16:35,966 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 66e19d6a9a9885e5b2c786f2e4c45a8e 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-26T19:16:35,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66e19d6a9a9885e5b2c786f2e4c45a8e, store=A 2024-11-26T19:16:35,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T19:16:35,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66e19d6a9a9885e5b2c786f2e4c45a8e, store=B 2024-11-26T19:16:35,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T19:16:35,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66e19d6a9a9885e5b2c786f2e4c45a8e, store=C 2024-11-26T19:16:35,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T19:16:35,973 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112667a01ddf085642a090509436bfd920c6_66e19d6a9a9885e5b2c786f2e4c45a8e is 50, key is test_row_0/A:col10/1732648595964/Put/seqid=0 2024-11-26T19:16:35,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742386_1562 (size=12304) 2024-11-26T19:16:35,976 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:35,979 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112667a01ddf085642a090509436bfd920c6_66e19d6a9a9885e5b2c786f2e4c45a8e to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112667a01ddf085642a090509436bfd920c6_66e19d6a9a9885e5b2c786f2e4c45a8e 2024-11-26T19:16:35,980 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/A/0ddccb6d4dfc4b54b2c508ce52bf5713, store: [table=TestAcidGuarantees family=A region=66e19d6a9a9885e5b2c786f2e4c45a8e] 2024-11-26T19:16:35,981 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/A/0ddccb6d4dfc4b54b2c508ce52bf5713 is 175, key is test_row_0/A:col10/1732648595964/Put/seqid=0 2024-11-26T19:16:35,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742387_1563 (size=31105) 2024-11-26T19:16:35,986 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:35,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35416 deadline: 1732648655980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:35,986 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:35,986 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:35,986 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:35,992 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:35,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35456 deadline: 1732648655985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:35,993 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:35,993 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:35,993 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:35,993 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:35,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35426 deadline: 1732648655986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:35,993 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:35,993 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:35,993 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:35,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35454 deadline: 1732648655987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:35,993 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:35,993 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:35,993 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:35,993 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:36,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-26T19:16:36,062 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T19:16:36,062 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T19:16:36,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees 2024-11-26T19:16:36,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-26T19:16:36,064 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T19:16:36,065 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T19:16:36,065 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T19:16:36,096 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:36,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35416 deadline: 1732648656092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:36,097 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:36,097 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:36,097 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:36,105 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:36,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35456 deadline: 1732648656101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:36,105 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:36,105 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:36,105 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:36,105 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:36,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35426 deadline: 1732648656101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:36,106 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:36,106 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:36,106 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:36,106 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:36,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35454 deadline: 1732648656102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:36,106 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:36,106 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:36,106 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:36,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-26T19:16:36,217 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-26T19:16:36,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e. 2024-11-26T19:16:36,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e. as already flushing 2024-11-26T19:16:36,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e. 2024-11-26T19:16:36,217 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 66e19d6a9a9885e5b2c786f2e4c45a8e, NAME => 'TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:16:36,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 66e19d6a9a9885e5b2c786f2e4c45a8e, NAME => 'TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:16:36,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66e19d6a9a9885e5b2c786f2e4c45a8e, NAME => 'TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66e19d6a9a9885e5b2c786f2e4c45a8e, NAME => 'TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:16:36,303 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:36,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35416 deadline: 1732648656301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:36,303 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:36,303 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:36,303 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:36,315 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:36,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35456 deadline: 1732648656311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:36,315 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:36,315 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:36,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35426 deadline: 1732648656312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:36,315 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:36,315 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:36,315 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:36,315 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:36,315 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:36,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35454 deadline: 1732648656312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:36,315 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:36,316 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:36,316 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:36,316 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:36,370 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-26T19:16:36,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e. 2024-11-26T19:16:36,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e. as already flushing 2024-11-26T19:16:36,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e. 2024-11-26T19:16:36,370 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 66e19d6a9a9885e5b2c786f2e4c45a8e, NAME => 'TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:16:36,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 66e19d6a9a9885e5b2c786f2e4c45a8e, NAME => 'TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:16:36,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66e19d6a9a9885e5b2c786f2e4c45a8e, NAME => 'TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66e19d6a9a9885e5b2c786f2e4c45a8e, NAME => 'TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:16:36,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-26T19:16:36,384 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=196, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/A/0ddccb6d4dfc4b54b2c508ce52bf5713 2024-11-26T19:16:36,391 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/B/b4ccec93922647e982907ebd40a6a66a is 50, key is test_row_0/B:col10/1732648595964/Put/seqid=0 2024-11-26T19:16:36,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742388_1564 (size=12151) 2024-11-26T19:16:36,396 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/B/b4ccec93922647e982907ebd40a6a66a 2024-11-26T19:16:36,403 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/C/dbc0a37b13b547108005db1715274e7e is 50, key is test_row_0/C:col10/1732648595964/Put/seqid=0 2024-11-26T19:16:36,405 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742389_1565 (size=12151) 2024-11-26T19:16:36,445 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:36,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35438 deadline: 1732648656442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:36,445 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:36,445 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:36,445 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:36,523 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-26T19:16:36,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e. 2024-11-26T19:16:36,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e. as already flushing 2024-11-26T19:16:36,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e. 2024-11-26T19:16:36,523 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 66e19d6a9a9885e5b2c786f2e4c45a8e, NAME => 'TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:16:36,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 66e19d6a9a9885e5b2c786f2e4c45a8e, NAME => 'TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:16:36,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66e19d6a9a9885e5b2c786f2e4c45a8e, NAME => 'TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66e19d6a9a9885e5b2c786f2e4c45a8e, NAME => 'TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:16:36,617 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:36,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35416 deadline: 1732648656612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:36,618 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:36,618 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:36,618 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:36,624 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:36,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35456 deadline: 1732648656621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:36,625 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:36,625 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:36,625 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:36,625 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:36,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35426 deadline: 1732648656621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:36,625 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:36,625 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:36,625 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:36,625 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:36,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35454 deadline: 1732648656622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:36,625 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:36,625 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:36,625 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:36,676 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-26T19:16:36,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e. 2024-11-26T19:16:36,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e. as already flushing 2024-11-26T19:16:36,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e. 2024-11-26T19:16:36,676 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 66e19d6a9a9885e5b2c786f2e4c45a8e, NAME => 'TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:16:36,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 66e19d6a9a9885e5b2c786f2e4c45a8e, NAME => 'TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:16:36,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66e19d6a9a9885e5b2c786f2e4c45a8e, NAME => 'TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66e19d6a9a9885e5b2c786f2e4c45a8e, NAME => 'TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:16:36,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-26T19:16:36,806 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/C/dbc0a37b13b547108005db1715274e7e 2024-11-26T19:16:36,810 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/A/0ddccb6d4dfc4b54b2c508ce52bf5713 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/0ddccb6d4dfc4b54b2c508ce52bf5713 2024-11-26T19:16:36,813 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/0ddccb6d4dfc4b54b2c508ce52bf5713, entries=150, sequenceid=196, filesize=30.4 K 2024-11-26T19:16:36,814 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/B/b4ccec93922647e982907ebd40a6a66a as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/B/b4ccec93922647e982907ebd40a6a66a 2024-11-26T19:16:36,818 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/B/b4ccec93922647e982907ebd40a6a66a, entries=150, sequenceid=196, filesize=11.9 K 2024-11-26T19:16:36,818 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/C/dbc0a37b13b547108005db1715274e7e as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/C/dbc0a37b13b547108005db1715274e7e 2024-11-26T19:16:36,822 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/C/dbc0a37b13b547108005db1715274e7e, entries=150, sequenceid=196, filesize=11.9 K 2024-11-26T19:16:36,823 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=53.67 KB/54960 for 66e19d6a9a9885e5b2c786f2e4c45a8e in 856ms, sequenceid=196, compaction requested=true 2024-11-26T19:16:36,823 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 66e19d6a9a9885e5b2c786f2e4c45a8e: 2024-11-26T19:16:36,823 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 66e19d6a9a9885e5b2c786f2e4c45a8e:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T19:16:36,823 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:16:36,823 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T19:16:36,823 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 66e19d6a9a9885e5b2c786f2e4c45a8e:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T19:16:36,823 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:16:36,823 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 66e19d6a9a9885e5b2c786f2e4c45a8e:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T19:16:36,823 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T19:16:36,823 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T19:16:36,824 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T19:16:36,824 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93657 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T19:16:36,824 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1541): 66e19d6a9a9885e5b2c786f2e4c45a8e/A is initiating minor compaction (all files) 2024-11-26T19:16:36,824 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1541): 66e19d6a9a9885e5b2c786f2e4c45a8e/B is initiating minor compaction (all files) 2024-11-26T19:16:36,824 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 66e19d6a9a9885e5b2c786f2e4c45a8e/A in TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e. 2024-11-26T19:16:36,824 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 66e19d6a9a9885e5b2c786f2e4c45a8e/B in TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e. 2024-11-26T19:16:36,824 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/bb50fc14458e4ff493f25015fe5a5e0f, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/d9629b526b2840e29e5f097c0c495118, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/0ddccb6d4dfc4b54b2c508ce52bf5713] into tmpdir=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp, totalSize=91.5 K 2024-11-26T19:16:36,824 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/B/cd980fc48f114e9ea2f096da0d4a4d09, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/B/db4363f29ea240e9920f9452c1c0b996, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/B/b4ccec93922647e982907ebd40a6a66a] into tmpdir=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp, totalSize=35.9 K 2024-11-26T19:16:36,824 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e. 2024-11-26T19:16:36,824 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e. files: [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/bb50fc14458e4ff493f25015fe5a5e0f, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/d9629b526b2840e29e5f097c0c495118, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/0ddccb6d4dfc4b54b2c508ce52bf5713] 2024-11-26T19:16:36,825 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting bb50fc14458e4ff493f25015fe5a5e0f, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1732648593492 2024-11-26T19:16:36,825 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting cd980fc48f114e9ea2f096da0d4a4d09, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1732648593492 2024-11-26T19:16:36,825 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting d9629b526b2840e29e5f097c0c495118, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1732648594139 2024-11-26T19:16:36,825 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting db4363f29ea240e9920f9452c1c0b996, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1732648594139 2024-11-26T19:16:36,825 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting b4ccec93922647e982907ebd40a6a66a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1732648595318 2024-11-26T19:16:36,825 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0ddccb6d4dfc4b54b2c508ce52bf5713, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1732648595318 2024-11-26T19:16:36,828 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-26T19:16:36,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e. 2024-11-26T19:16:36,829 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2902): Flushing 66e19d6a9a9885e5b2c786f2e4c45a8e 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-26T19:16:36,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66e19d6a9a9885e5b2c786f2e4c45a8e, store=A 2024-11-26T19:16:36,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:16:36,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66e19d6a9a9885e5b2c786f2e4c45a8e, store=B 2024-11-26T19:16:36,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:16:36,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66e19d6a9a9885e5b2c786f2e4c45a8e, store=C 2024-11-26T19:16:36,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:16:36,831 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=66e19d6a9a9885e5b2c786f2e4c45a8e] 2024-11-26T19:16:36,832 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 66e19d6a9a9885e5b2c786f2e4c45a8e#B#compaction#476 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T19:16:36,833 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/B/33ffa57def274e60a37aeab3a7445f3c is 50, key is test_row_0/B:col10/1732648595964/Put/seqid=0 2024-11-26T19:16:36,834 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411261e170dfb2c0f48358b5f96e4ffc6d75b_66e19d6a9a9885e5b2c786f2e4c45a8e store=[table=TestAcidGuarantees family=A region=66e19d6a9a9885e5b2c786f2e4c45a8e] 2024-11-26T19:16:36,836 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411261e170dfb2c0f48358b5f96e4ffc6d75b_66e19d6a9a9885e5b2c786f2e4c45a8e, store=[table=TestAcidGuarantees family=A region=66e19d6a9a9885e5b2c786f2e4c45a8e] 2024-11-26T19:16:36,836 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411261e170dfb2c0f48358b5f96e4ffc6d75b_66e19d6a9a9885e5b2c786f2e4c45a8e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=66e19d6a9a9885e5b2c786f2e4c45a8e] 2024-11-26T19:16:36,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411266d64842c8a034640bbe8dfa78db60bcb_66e19d6a9a9885e5b2c786f2e4c45a8e is 50, key is test_row_0/A:col10/1732648595985/Put/seqid=0 2024-11-26T19:16:36,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742390_1566 (size=12595) 2024-11-26T19:16:36,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742391_1567 (size=4469) 2024-11-26T19:16:36,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742392_1568 (size=12304) 2024-11-26T19:16:37,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(8855): Flush requested on 66e19d6a9a9885e5b2c786f2e4c45a8e 2024-11-26T19:16:37,135 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e. as already flushing 2024-11-26T19:16:37,161 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:37,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35454 deadline: 1732648657156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:37,162 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:37,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35426 deadline: 1732648657157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:37,162 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:37,162 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:37,162 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:37,164 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732648597148, max timestamp=1732648597160Num uniques 859; 2024-11-26T19:16:37,164 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732648597148, max timestamp=1732648597160Num uniques 859; 2024-11-26T19:16:37,164 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-26T19:16:37,164 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-26T19:16:37,164 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732648597148, max timestamp=1732648597160Num uniques 859; 2024-11-26T19:16:37,164 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-26T19:16:37,164 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:37,164 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:37,164 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:37,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-26T19:16:37,213 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:37,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35456 deadline: 1732648657208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:37,214 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:37,214 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:37,214 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:37,216 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:37,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35416 deadline: 1732648657211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:37,216 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:37,216 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:37,216 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:37,241 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 66e19d6a9a9885e5b2c786f2e4c45a8e#A#compaction#477 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T19:16:37,242 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/A/bf7e8f9b4a7048d0b9cdfd192ee0f77a is 175, key is test_row_0/A:col10/1732648595964/Put/seqid=0 2024-11-26T19:16:37,245 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/B/33ffa57def274e60a37aeab3a7445f3c as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/B/33ffa57def274e60a37aeab3a7445f3c 2024-11-26T19:16:37,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742393_1569 (size=31549) 2024-11-26T19:16:37,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:37,251 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 66e19d6a9a9885e5b2c786f2e4c45a8e/B of 66e19d6a9a9885e5b2c786f2e4c45a8e into 33ffa57def274e60a37aeab3a7445f3c(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T19:16:37,251 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 66e19d6a9a9885e5b2c786f2e4c45a8e: 2024-11-26T19:16:37,251 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., storeName=66e19d6a9a9885e5b2c786f2e4c45a8e/B, priority=13, startTime=1732648596823; duration=0sec 2024-11-26T19:16:37,251 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T19:16:37,251 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 66e19d6a9a9885e5b2c786f2e4c45a8e:B 2024-11-26T19:16:37,251 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411266d64842c8a034640bbe8dfa78db60bcb_66e19d6a9a9885e5b2c786f2e4c45a8e to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411266d64842c8a034640bbe8dfa78db60bcb_66e19d6a9a9885e5b2c786f2e4c45a8e 2024-11-26T19:16:37,251 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T19:16:37,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/A/ee38a39e45ca41c4ba673e6ea07bbb38, store: [table=TestAcidGuarantees family=A region=66e19d6a9a9885e5b2c786f2e4c45a8e] 2024-11-26T19:16:37,252 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T19:16:37,252 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1541): 66e19d6a9a9885e5b2c786f2e4c45a8e/C is initiating minor compaction (all files) 2024-11-26T19:16:37,252 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 66e19d6a9a9885e5b2c786f2e4c45a8e/C in TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e. 2024-11-26T19:16:37,252 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/C/c99db25f98c64c4a878c80e1720d6611, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/C/2de53e2a3cd84482b3e52ec82eda114f, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/C/dbc0a37b13b547108005db1715274e7e] into tmpdir=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp, totalSize=35.9 K 2024-11-26T19:16:37,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/A/ee38a39e45ca41c4ba673e6ea07bbb38 is 175, key is test_row_0/A:col10/1732648595985/Put/seqid=0 2024-11-26T19:16:37,253 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting c99db25f98c64c4a878c80e1720d6611, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1732648593492 2024-11-26T19:16:37,254 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting 2de53e2a3cd84482b3e52ec82eda114f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1732648594139 2024-11-26T19:16:37,254 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting dbc0a37b13b547108005db1715274e7e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1732648595318 2024-11-26T19:16:37,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742394_1570 (size=31105) 2024-11-26T19:16:37,262 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=207, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/A/ee38a39e45ca41c4ba673e6ea07bbb38 2024-11-26T19:16:37,262 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 66e19d6a9a9885e5b2c786f2e4c45a8e#C#compaction#479 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T19:16:37,263 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/C/007764b90f694955baae50481b1155f2 is 50, key is test_row_0/C:col10/1732648595964/Put/seqid=0 2024-11-26T19:16:37,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742395_1571 (size=12595) 2024-11-26T19:16:37,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/B/033ab683305b46dbb50c21e56c93ac94 is 50, key is test_row_0/B:col10/1732648595985/Put/seqid=0 2024-11-26T19:16:37,278 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:37,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35454 deadline: 1732648657272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:37,279 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:37,279 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:37,279 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:37,279 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:37,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35426 deadline: 1732648657272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:37,279 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:37,279 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:37,279 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:37,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742396_1572 (size=12151) 2024-11-26T19:16:37,325 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:37,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35456 deadline: 1732648657322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:37,325 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:37,325 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:37,325 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:37,328 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:37,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35416 deadline: 1732648657322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:37,328 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:37,328 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:37,328 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:37,456 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:37,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35438 deadline: 1732648657452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:37,457 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to fe76acf732b2,42785,1732648423970 for 'test_row_2' in 66e19d6a9a9885e5b2c786f2e4c45a8e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2153 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T19:16:37,457 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:37,457 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:37,457 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:37,495 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:37,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35454 deadline: 1732648657491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:37,495 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:37,495 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:37,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35426 deadline: 1732648657491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:37,495 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:37,495 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:37,495 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:37,495 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:37,495 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:37,536 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:37,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35456 deadline: 1732648657532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:37,537 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:37,537 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:37,537 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:37,537 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:37,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35416 deadline: 1732648657532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:37,537 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:37,537 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:37,537 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:37,651 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/A/bf7e8f9b4a7048d0b9cdfd192ee0f77a as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/bf7e8f9b4a7048d0b9cdfd192ee0f77a 2024-11-26T19:16:37,654 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 66e19d6a9a9885e5b2c786f2e4c45a8e/A of 66e19d6a9a9885e5b2c786f2e4c45a8e into bf7e8f9b4a7048d0b9cdfd192ee0f77a(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T19:16:37,654 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 66e19d6a9a9885e5b2c786f2e4c45a8e: 2024-11-26T19:16:37,654 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., storeName=66e19d6a9a9885e5b2c786f2e4c45a8e/A, priority=13, startTime=1732648596823; duration=0sec 2024-11-26T19:16:37,655 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:16:37,655 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 66e19d6a9a9885e5b2c786f2e4c45a8e:A 2024-11-26T19:16:37,673 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/C/007764b90f694955baae50481b1155f2 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/C/007764b90f694955baae50481b1155f2 2024-11-26T19:16:37,677 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 66e19d6a9a9885e5b2c786f2e4c45a8e/C of 66e19d6a9a9885e5b2c786f2e4c45a8e into 007764b90f694955baae50481b1155f2(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T19:16:37,677 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 66e19d6a9a9885e5b2c786f2e4c45a8e: 2024-11-26T19:16:37,677 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., storeName=66e19d6a9a9885e5b2c786f2e4c45a8e/C, priority=13, startTime=1732648596823; duration=0sec 2024-11-26T19:16:37,678 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:16:37,678 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 66e19d6a9a9885e5b2c786f2e4c45a8e:C 2024-11-26T19:16:37,682 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/B/033ab683305b46dbb50c21e56c93ac94 2024-11-26T19:16:37,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/C/f45532affc4342e086eb0733c882787b is 50, key is test_row_0/C:col10/1732648595985/Put/seqid=0 2024-11-26T19:16:37,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742397_1573 (size=12151) 2024-11-26T19:16:37,807 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:37,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35454 deadline: 1732648657802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:37,807 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:37,807 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:37,807 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:37,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35426 deadline: 1732648657802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:37,807 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:37,807 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:37,807 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:37,807 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:37,846 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:37,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35456 deadline: 1732648657842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:37,846 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:37,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35416 deadline: 1732648657842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:37,846 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:37,846 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:37,846 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:37,846 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:37,846 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:37,846 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:38,093 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/C/f45532affc4342e086eb0733c882787b 2024-11-26T19:16:38,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/A/ee38a39e45ca41c4ba673e6ea07bbb38 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/ee38a39e45ca41c4ba673e6ea07bbb38 2024-11-26T19:16:38,100 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/ee38a39e45ca41c4ba673e6ea07bbb38, entries=150, sequenceid=207, filesize=30.4 K 2024-11-26T19:16:38,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/B/033ab683305b46dbb50c21e56c93ac94 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/B/033ab683305b46dbb50c21e56c93ac94 2024-11-26T19:16:38,104 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/B/033ab683305b46dbb50c21e56c93ac94, entries=150, sequenceid=207, filesize=11.9 K 2024-11-26T19:16:38,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/C/f45532affc4342e086eb0733c882787b as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/C/f45532affc4342e086eb0733c882787b 2024-11-26T19:16:38,107 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/C/f45532affc4342e086eb0733c882787b, entries=150, sequenceid=207, filesize=11.9 K 2024-11-26T19:16:38,108 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 66e19d6a9a9885e5b2c786f2e4c45a8e in 1279ms, sequenceid=207, compaction requested=false 2024-11-26T19:16:38,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2603): Flush status journal for 66e19d6a9a9885e5b2c786f2e4c45a8e: 2024-11-26T19:16:38,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e. 2024-11-26T19:16:38,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=155 2024-11-26T19:16:38,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster(4169): Remote procedure done, pid=155 2024-11-26T19:16:38,110 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-11-26T19:16:38,111 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0440 sec 2024-11-26T19:16:38,112 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees in 2.0490 sec 2024-11-26T19:16:38,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-26T19:16:38,212 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T19:16:38,212 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T19:16:38,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees 2024-11-26T19:16:38,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-26T19:16:38,214 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T19:16:38,215 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T19:16:38,215 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T19:16:38,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(8855): Flush requested on 66e19d6a9a9885e5b2c786f2e4c45a8e 2024-11-26T19:16:38,319 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 66e19d6a9a9885e5b2c786f2e4c45a8e 3/3 column families, dataSize=140.89 KB heapSize=405.61 KB 2024-11-26T19:16:38,319 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66e19d6a9a9885e5b2c786f2e4c45a8e, store=A 2024-11-26T19:16:38,319 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-26T19:16:38,319 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66e19d6a9a9885e5b2c786f2e4c45a8e, store=B 2024-11-26T19:16:38,319 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-26T19:16:38,319 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66e19d6a9a9885e5b2c786f2e4c45a8e, store=C 2024-11-26T19:16:38,319 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-26T19:16:38,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-26T19:16:38,332 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411263b43cabc3a724edda5eef4e6eb058372_66e19d6a9a9885e5b2c786f2e4c45a8e is 50, key is test_row_0/A:col10/1732648597209/Put/seqid=0 2024-11-26T19:16:38,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742398_1574 (size=17284) 2024-11-26T19:16:38,341 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:38,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35454 deadline: 1732648658337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:38,341 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:38,341 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:38,341 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:38,344 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:38,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35426 deadline: 1732648658338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:38,344 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:38,345 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:38,345 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:38,355 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:38,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35456 deadline: 1732648658352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:38,356 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:38,356 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:38,356 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:38,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35416 deadline: 1732648658352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:38,356 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:38,356 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:38,356 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:38,356 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:38,367 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-26T19:16:38,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e. 2024-11-26T19:16:38,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e. as already flushing 2024-11-26T19:16:38,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e. 2024-11-26T19:16:38,367 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 66e19d6a9a9885e5b2c786f2e4c45a8e, NAME => 'TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:16:38,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 66e19d6a9a9885e5b2c786f2e4c45a8e, NAME => 'TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:16:38,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66e19d6a9a9885e5b2c786f2e4c45a8e, NAME => 'TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66e19d6a9a9885e5b2c786f2e4c45a8e, NAME => 'TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:16:38,456 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:38,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35454 deadline: 1732648658452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:38,456 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:38,456 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:38,456 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:38,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35426 deadline: 1732648658452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:38,456 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:38,456 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:38,457 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:38,457 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:38,520 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-26T19:16:38,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e. 2024-11-26T19:16:38,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e. as already flushing 2024-11-26T19:16:38,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e. 2024-11-26T19:16:38,520 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 66e19d6a9a9885e5b2c786f2e4c45a8e, NAME => 'TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:16:38,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 66e19d6a9a9885e5b2c786f2e4c45a8e, NAME => 'TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:16:38,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66e19d6a9a9885e5b2c786f2e4c45a8e, NAME => 'TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66e19d6a9a9885e5b2c786f2e4c45a8e, NAME => 'TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:16:38,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-26T19:16:38,666 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:38,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35454 deadline: 1732648658662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:38,667 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:38,667 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:38,667 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:38,667 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:38,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35426 deadline: 1732648658662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:38,668 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:38,668 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:38,668 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:38,673 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-26T19:16:38,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e. 2024-11-26T19:16:38,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e. as already flushing 2024-11-26T19:16:38,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e. 2024-11-26T19:16:38,673 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 66e19d6a9a9885e5b2c786f2e4c45a8e, NAME => 'TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:16:38,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 66e19d6a9a9885e5b2c786f2e4c45a8e, NAME => 'TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:16:38,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66e19d6a9a9885e5b2c786f2e4c45a8e, NAME => 'TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66e19d6a9a9885e5b2c786f2e4c45a8e, NAME => 'TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:16:38,736 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:38,739 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411263b43cabc3a724edda5eef4e6eb058372_66e19d6a9a9885e5b2c786f2e4c45a8e to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411263b43cabc3a724edda5eef4e6eb058372_66e19d6a9a9885e5b2c786f2e4c45a8e 2024-11-26T19:16:38,740 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/A/076c35a352c64be0a3cda5c12aa5731b, store: [table=TestAcidGuarantees family=A region=66e19d6a9a9885e5b2c786f2e4c45a8e] 2024-11-26T19:16:38,740 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/A/076c35a352c64be0a3cda5c12aa5731b is 175, key is test_row_0/A:col10/1732648597209/Put/seqid=0 2024-11-26T19:16:38,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742399_1575 (size=48389) 2024-11-26T19:16:38,825 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-26T19:16:38,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e. 2024-11-26T19:16:38,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e. as already flushing 2024-11-26T19:16:38,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e. 2024-11-26T19:16:38,826 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 66e19d6a9a9885e5b2c786f2e4c45a8e, NAME => 'TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:16:38,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 66e19d6a9a9885e5b2c786f2e4c45a8e, NAME => 'TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:16:38,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66e19d6a9a9885e5b2c786f2e4c45a8e, NAME => 'TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66e19d6a9a9885e5b2c786f2e4c45a8e, NAME => 'TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:16:38,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-26T19:16:38,976 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:38,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35454 deadline: 1732648658971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:38,976 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:38,976 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:38,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35426 deadline: 1732648658971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:38,976 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:38,976 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:38,977 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:38,977 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:38,977 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:38,978 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-26T19:16:38,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e. 2024-11-26T19:16:38,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e. as already flushing 2024-11-26T19:16:38,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e. 2024-11-26T19:16:38,979 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 66e19d6a9a9885e5b2c786f2e4c45a8e, NAME => 'TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:16:38,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 66e19d6a9a9885e5b2c786f2e4c45a8e, NAME => 'TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:16:38,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66e19d6a9a9885e5b2c786f2e4c45a8e, NAME => 'TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66e19d6a9a9885e5b2c786f2e4c45a8e, NAME => 'TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:16:39,131 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-26T19:16:39,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e. 2024-11-26T19:16:39,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e. as already flushing 2024-11-26T19:16:39,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e. 2024-11-26T19:16:39,131 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 66e19d6a9a9885e5b2c786f2e4c45a8e, NAME => 'TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:16:39,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 66e19d6a9a9885e5b2c786f2e4c45a8e, NAME => 'TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:16:39,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66e19d6a9a9885e5b2c786f2e4c45a8e, NAME => 'TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66e19d6a9a9885e5b2c786f2e4c45a8e, NAME => 'TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:16:39,144 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=248, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/A/076c35a352c64be0a3cda5c12aa5731b 2024-11-26T19:16:39,152 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/B/d549d8f97b66450ea46484f502fc53e8 is 50, key is test_row_0/B:col10/1732648597209/Put/seqid=0 2024-11-26T19:16:39,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742400_1576 (size=12151) 2024-11-26T19:16:39,284 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-26T19:16:39,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e. 2024-11-26T19:16:39,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e. as already flushing 2024-11-26T19:16:39,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e. 2024-11-26T19:16:39,284 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 66e19d6a9a9885e5b2c786f2e4c45a8e, NAME => 'TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:16:39,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 66e19d6a9a9885e5b2c786f2e4c45a8e, NAME => 'TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:16:39,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66e19d6a9a9885e5b2c786f2e4c45a8e, NAME => 'TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66e19d6a9a9885e5b2c786f2e4c45a8e, NAME => 'TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:16:39,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-26T19:16:39,366 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:39,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35456 deadline: 1732648659361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:39,367 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:39,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35416 deadline: 1732648659361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:39,367 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to fe76acf732b2,42785,1732648423970 for 'test_row_2' in 66e19d6a9a9885e5b2c786f2e4c45a8e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2159 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T19:16:39,368 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:39,368 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:39,368 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:39,367 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to fe76acf732b2,42785,1732648423970 for 'test_row_0' in 66e19d6a9a9885e5b2c786f2e4c45a8e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2156 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T19:16:39,368 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:39,368 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:39,368 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:39,437 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-26T19:16:39,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e. 2024-11-26T19:16:39,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e. as already flushing 2024-11-26T19:16:39,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e. 2024-11-26T19:16:39,438 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 66e19d6a9a9885e5b2c786f2e4c45a8e, NAME => 'TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:16:39,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 66e19d6a9a9885e5b2c786f2e4c45a8e, NAME => 'TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:16:39,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66e19d6a9a9885e5b2c786f2e4c45a8e, NAME => 'TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66e19d6a9a9885e5b2c786f2e4c45a8e, NAME => 'TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:16:39,472 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:39,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35438 deadline: 1732648659471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:39,473 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to fe76acf732b2,42785,1732648423970 for 'test_row_2' in 66e19d6a9a9885e5b2c786f2e4c45a8e of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T19:16:39,474 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:39,474 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:39,474 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:39,485 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:39,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35454 deadline: 1732648659481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:39,486 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:39,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35426 deadline: 1732648659482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:39,486 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:39,486 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:39,486 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:39,486 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:39,486 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:39,486 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:39,557 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/B/d549d8f97b66450ea46484f502fc53e8 2024-11-26T19:16:39,565 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/C/6112da08b8ca41819ed10ac00fc90ef4 is 50, key is test_row_0/C:col10/1732648597209/Put/seqid=0 2024-11-26T19:16:39,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742401_1577 (size=12151) 2024-11-26T19:16:39,573 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/C/6112da08b8ca41819ed10ac00fc90ef4 2024-11-26T19:16:39,577 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/A/076c35a352c64be0a3cda5c12aa5731b as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/076c35a352c64be0a3cda5c12aa5731b 2024-11-26T19:16:39,580 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/076c35a352c64be0a3cda5c12aa5731b, entries=250, sequenceid=248, filesize=47.3 K 2024-11-26T19:16:39,580 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/B/d549d8f97b66450ea46484f502fc53e8 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/B/d549d8f97b66450ea46484f502fc53e8 2024-11-26T19:16:39,583 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/B/d549d8f97b66450ea46484f502fc53e8, entries=150, sequenceid=248, filesize=11.9 K 2024-11-26T19:16:39,584 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/C/6112da08b8ca41819ed10ac00fc90ef4 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/C/6112da08b8ca41819ed10ac00fc90ef4 2024-11-26T19:16:39,586 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/C/6112da08b8ca41819ed10ac00fc90ef4, entries=150, sequenceid=248, filesize=11.9 K 2024-11-26T19:16:39,587 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~405.56 KB/415296, currentSize=40.25 KB/41220 for 66e19d6a9a9885e5b2c786f2e4c45a8e in 1268ms, sequenceid=248, compaction requested=true 2024-11-26T19:16:39,587 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 66e19d6a9a9885e5b2c786f2e4c45a8e: 2024-11-26T19:16:39,587 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 66e19d6a9a9885e5b2c786f2e4c45a8e:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T19:16:39,587 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:16:39,587 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T19:16:39,587 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 66e19d6a9a9885e5b2c786f2e4c45a8e:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T19:16:39,587 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T19:16:39,587 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 66e19d6a9a9885e5b2c786f2e4c45a8e:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T19:16:39,587 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-26T19:16:39,588 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T19:16:39,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,589 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111043 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T19:16:39,589 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T19:16:39,589 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1541): 66e19d6a9a9885e5b2c786f2e4c45a8e/A is initiating minor compaction (all files) 2024-11-26T19:16:39,589 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1541): 66e19d6a9a9885e5b2c786f2e4c45a8e/B is initiating minor compaction (all files) 2024-11-26T19:16:39,589 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 66e19d6a9a9885e5b2c786f2e4c45a8e/A in TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e. 2024-11-26T19:16:39,589 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 66e19d6a9a9885e5b2c786f2e4c45a8e/B in TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e. 2024-11-26T19:16:39,589 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/bf7e8f9b4a7048d0b9cdfd192ee0f77a, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/ee38a39e45ca41c4ba673e6ea07bbb38, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/076c35a352c64be0a3cda5c12aa5731b] into tmpdir=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp, totalSize=108.4 K 2024-11-26T19:16:39,589 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/B/33ffa57def274e60a37aeab3a7445f3c, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/B/033ab683305b46dbb50c21e56c93ac94, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/B/d549d8f97b66450ea46484f502fc53e8] into tmpdir=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp, totalSize=36.0 K 2024-11-26T19:16:39,589 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e. 2024-11-26T19:16:39,589 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e. files: [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/bf7e8f9b4a7048d0b9cdfd192ee0f77a, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/ee38a39e45ca41c4ba673e6ea07bbb38, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/076c35a352c64be0a3cda5c12aa5731b] 2024-11-26T19:16:39,589 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting bf7e8f9b4a7048d0b9cdfd192ee0f77a, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1732648595318 2024-11-26T19:16:39,589 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting 33ffa57def274e60a37aeab3a7445f3c, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1732648595318 2024-11-26T19:16:39,590 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting 033ab683305b46dbb50c21e56c93ac94, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1732648595974 2024-11-26T19:16:39,590 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting ee38a39e45ca41c4ba673e6ea07bbb38, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1732648595974 2024-11-26T19:16:39,590 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting d549d8f97b66450ea46484f502fc53e8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1732648597209 2024-11-26T19:16:39,590 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting 076c35a352c64be0a3cda5c12aa5731b, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1732648597196 2024-11-26T19:16:39,590 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-26T19:16:39,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e. 2024-11-26T19:16:39,591 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2902): Flushing 66e19d6a9a9885e5b2c786f2e4c45a8e 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-26T19:16:39,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66e19d6a9a9885e5b2c786f2e4c45a8e, store=A 2024-11-26T19:16:39,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:16:39,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66e19d6a9a9885e5b2c786f2e4c45a8e, store=B 2024-11-26T19:16:39,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:16:39,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66e19d6a9a9885e5b2c786f2e4c45a8e, store=C 2024-11-26T19:16:39,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:16:39,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112619b586e2bfb84315a264b2d755b47639_66e19d6a9a9885e5b2c786f2e4c45a8e is 50, key is test_row_0/A:col10/1732648598337/Put/seqid=0 2024-11-26T19:16:39,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,598 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 66e19d6a9a9885e5b2c786f2e4c45a8e#B#compaction#486 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T19:16:39,598 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/B/e44547b5e20744f596ab8b11b0bcd794 is 50, key is test_row_0/B:col10/1732648597209/Put/seqid=0 2024-11-26T19:16:39,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,599 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=66e19d6a9a9885e5b2c786f2e4c45a8e] 2024-11-26T19:16:39,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,608 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241126c901519963854486be9028ddd08d0efa_66e19d6a9a9885e5b2c786f2e4c45a8e store=[table=TestAcidGuarantees family=A region=66e19d6a9a9885e5b2c786f2e4c45a8e] 2024-11-26T19:16:39,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,610 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241126c901519963854486be9028ddd08d0efa_66e19d6a9a9885e5b2c786f2e4c45a8e, store=[table=TestAcidGuarantees family=A region=66e19d6a9a9885e5b2c786f2e4c45a8e] 2024-11-26T19:16:39,610 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126c901519963854486be9028ddd08d0efa_66e19d6a9a9885e5b2c786f2e4c45a8e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=66e19d6a9a9885e5b2c786f2e4c45a8e] 2024-11-26T19:16:39,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742402_1578 (size=9814) 2024-11-26T19:16:39,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742403_1579 (size=12697) 2024-11-26T19:16:39,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742404_1580 (size=4469) 2024-11-26T19:16:39,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:39,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,012 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112619b586e2bfb84315a264b2d755b47639_66e19d6a9a9885e5b2c786f2e4c45a8e to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112619b586e2bfb84315a264b2d755b47639_66e19d6a9a9885e5b2c786f2e4c45a8e 2024-11-26T19:16:40,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/A/2bf74ae587d042c9a08356c0ebca120f, store: [table=TestAcidGuarantees family=A region=66e19d6a9a9885e5b2c786f2e4c45a8e] 2024-11-26T19:16:40,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/A/2bf74ae587d042c9a08356c0ebca120f is 175, key is test_row_0/A:col10/1732648598337/Put/seqid=0 2024-11-26T19:16:40,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742405_1581 (size=22461) 2024-11-26T19:16:40,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,020 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 66e19d6a9a9885e5b2c786f2e4c45a8e#A#compaction#487 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T19:16:40,020 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/B/e44547b5e20744f596ab8b11b0bcd794 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/B/e44547b5e20744f596ab8b11b0bcd794 2024-11-26T19:16:40,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,021 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/A/46a2cd1070d14569b1f23ff22d1b6f9d is 175, key is test_row_0/A:col10/1732648597209/Put/seqid=0 2024-11-26T19:16:40,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742406_1582 (size=31651) 2024-11-26T19:16:40,025 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 66e19d6a9a9885e5b2c786f2e4c45a8e/B of 66e19d6a9a9885e5b2c786f2e4c45a8e into e44547b5e20744f596ab8b11b0bcd794(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T19:16:40,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,025 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 66e19d6a9a9885e5b2c786f2e4c45a8e: 2024-11-26T19:16:40,025 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., storeName=66e19d6a9a9885e5b2c786f2e4c45a8e/B, priority=13, startTime=1732648599587; duration=0sec 2024-11-26T19:16:40,025 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T19:16:40,025 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 66e19d6a9a9885e5b2c786f2e4c45a8e:B 2024-11-26T19:16:40,026 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T19:16:40,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,026 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T19:16:40,026 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1541): 66e19d6a9a9885e5b2c786f2e4c45a8e/C is initiating minor compaction (all files) 2024-11-26T19:16:40,027 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 66e19d6a9a9885e5b2c786f2e4c45a8e/C in TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e. 2024-11-26T19:16:40,027 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/C/007764b90f694955baae50481b1155f2, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/C/f45532affc4342e086eb0733c882787b, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/C/6112da08b8ca41819ed10ac00fc90ef4] into tmpdir=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp, totalSize=36.0 K 2024-11-26T19:16:40,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,027 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting 007764b90f694955baae50481b1155f2, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1732648595318 2024-11-26T19:16:40,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,027 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting f45532affc4342e086eb0733c882787b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1732648595974 2024-11-26T19:16:40,028 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting 6112da08b8ca41819ed10ac00fc90ef4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1732648597209 2024-11-26T19:16:40,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,034 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 66e19d6a9a9885e5b2c786f2e4c45a8e#C#compaction#488 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T19:16:40,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,034 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/C/f6aa479c4752431b8b11bc35a1dd2c7b is 50, key is test_row_0/C:col10/1732648597209/Put/seqid=0 2024-11-26T19:16:40,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742407_1583 (size=12697) 2024-11-26T19:16:40,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,042 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/C/f6aa479c4752431b8b11bc35a1dd2c7b as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/C/f6aa479c4752431b8b11bc35a1dd2c7b 2024-11-26T19:16:40,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,046 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 66e19d6a9a9885e5b2c786f2e4c45a8e/C of 66e19d6a9a9885e5b2c786f2e4c45a8e into f6aa479c4752431b8b11bc35a1dd2c7b(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T19:16:40,046 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 66e19d6a9a9885e5b2c786f2e4c45a8e: 2024-11-26T19:16:40,046 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., storeName=66e19d6a9a9885e5b2c786f2e4c45a8e/C, priority=13, startTime=1732648599587; duration=0sec 2024-11-26T19:16:40,046 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:16:40,046 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 66e19d6a9a9885e5b2c786f2e4c45a8e:C 2024-11-26T19:16:40,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-26T19:16:40,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,418 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=257, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/A/2bf74ae587d042c9a08356c0ebca120f 2024-11-26T19:16:40,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/B/6149ecd1bc264563af5527de7993cc86 is 50, key is test_row_0/B:col10/1732648598337/Put/seqid=0 2024-11-26T19:16:40,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,433 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/A/46a2cd1070d14569b1f23ff22d1b6f9d as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/46a2cd1070d14569b1f23ff22d1b6f9d 2024-11-26T19:16:40,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742408_1584 (size=9757) 2024-11-26T19:16:40,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,449 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 66e19d6a9a9885e5b2c786f2e4c45a8e/A of 66e19d6a9a9885e5b2c786f2e4c45a8e into 46a2cd1070d14569b1f23ff22d1b6f9d(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T19:16:40,449 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 66e19d6a9a9885e5b2c786f2e4c45a8e: 2024-11-26T19:16:40,449 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., storeName=66e19d6a9a9885e5b2c786f2e4c45a8e/A, priority=13, startTime=1732648599587; duration=0sec 2024-11-26T19:16:40,449 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:16:40,449 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 66e19d6a9a9885e5b2c786f2e4c45a8e:A 2024-11-26T19:16:40,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(8855): Flush requested on 66e19d6a9a9885e5b2c786f2e4c45a8e 2024-11-26T19:16:40,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,505 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e. as already flushing 2024-11-26T19:16:40,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:40,562 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732648600541, max timestamp=1732648600558Num uniques 859; 2024-11-26T19:16:40,562 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732648600541, max timestamp=1732648600558Num uniques 859; 2024-11-26T19:16:40,562 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-26T19:16:40,562 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-26T19:16:40,562 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732648600541, max timestamp=1732648600558Num uniques 859; 2024-11-26T19:16:40,562 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-26T19:16:40,615 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732648600582, max timestamp=1732648600610Num uniques 1068; 2024-11-26T19:16:40,615 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-11-26T19:16:40,615 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732648600582, max timestamp=1732648600610Num uniques 1068; 2024-11-26T19:16:40,615 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-11-26T19:16:40,615 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732648600582, max timestamp=1732648600610Num uniques 1068; 2024-11-26T19:16:40,616 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-11-26T19:16:40,665 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:40,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35426 deadline: 1732648660660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:40,665 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:40,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35454 deadline: 1732648660660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:40,666 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:40,666 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:40,666 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:40,666 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:40,666 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:40,666 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:40,774 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:40,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35426 deadline: 1732648660771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:40,774 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:40,774 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:40,774 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:40,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35454 deadline: 1732648660771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:40,774 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:40,774 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:40,774 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:40,774 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:40,834 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=257 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/B/6149ecd1bc264563af5527de7993cc86 2024-11-26T19:16:40,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/C/cb3a0b5aca7e4857b9d4ae30b3ed702e is 50, key is test_row_0/C:col10/1732648598337/Put/seqid=0 2024-11-26T19:16:40,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742409_1585 (size=9757) 2024-11-26T19:16:40,986 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:40,986 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:40,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35426 deadline: 1732648660982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:40,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35454 deadline: 1732648660982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:40,987 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:40,987 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:40,987 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:40,987 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:40,987 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:40,987 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:41,244 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=257 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/C/cb3a0b5aca7e4857b9d4ae30b3ed702e 2024-11-26T19:16:41,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/A/2bf74ae587d042c9a08356c0ebca120f as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/2bf74ae587d042c9a08356c0ebca120f 2024-11-26T19:16:41,251 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/2bf74ae587d042c9a08356c0ebca120f, entries=100, sequenceid=257, filesize=21.9 K 2024-11-26T19:16:41,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/B/6149ecd1bc264563af5527de7993cc86 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/B/6149ecd1bc264563af5527de7993cc86 2024-11-26T19:16:41,255 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/B/6149ecd1bc264563af5527de7993cc86, entries=100, sequenceid=257, filesize=9.5 K 2024-11-26T19:16:41,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/C/cb3a0b5aca7e4857b9d4ae30b3ed702e as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/C/cb3a0b5aca7e4857b9d4ae30b3ed702e 2024-11-26T19:16:41,258 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/C/cb3a0b5aca7e4857b9d4ae30b3ed702e, entries=100, sequenceid=257, filesize=9.5 K 2024-11-26T19:16:41,259 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=140.89 KB/144270 for 66e19d6a9a9885e5b2c786f2e4c45a8e in 1668ms, sequenceid=257, compaction requested=false 2024-11-26T19:16:41,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2603): Flush status journal for 66e19d6a9a9885e5b2c786f2e4c45a8e: 2024-11-26T19:16:41,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e. 2024-11-26T19:16:41,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=157 2024-11-26T19:16:41,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster(4169): Remote procedure done, pid=157 2024-11-26T19:16:41,262 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-11-26T19:16:41,262 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.0450 sec 2024-11-26T19:16:41,263 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees in 3.0500 sec 2024-11-26T19:16:41,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(8855): Flush requested on 66e19d6a9a9885e5b2c786f2e4c45a8e 2024-11-26T19:16:41,296 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 66e19d6a9a9885e5b2c786f2e4c45a8e 3/3 column families, dataSize=147.60 KB heapSize=420 KB 2024-11-26T19:16:41,296 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66e19d6a9a9885e5b2c786f2e4c45a8e, store=A 2024-11-26T19:16:41,296 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-26T19:16:41,296 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66e19d6a9a9885e5b2c786f2e4c45a8e, store=B 2024-11-26T19:16:41,296 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-26T19:16:41,296 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66e19d6a9a9885e5b2c786f2e4c45a8e, store=C 2024-11-26T19:16:41,296 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-26T19:16:41,306 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112696bda8669b7e4edbb0c47ae206e28f9b_66e19d6a9a9885e5b2c786f2e4c45a8e is 50, key is test_row_0/A:col10/1732648601294/Put/seqid=0 2024-11-26T19:16:41,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742410_1586 (size=12454) 2024-11-26T19:16:41,315 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:41,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35426 deadline: 1732648661310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:41,315 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:41,315 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:41,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35454 deadline: 1732648661311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:41,315 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:41,315 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:41,316 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:41,316 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:41,316 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:41,385 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:41,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35456 deadline: 1732648661382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:41,385 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to fe76acf732b2,42785,1732648423970 for 'test_row_2' in 66e19d6a9a9885e5b2c786f2e4c45a8e of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4177 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T19:16:41,386 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:41,386 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:41,386 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:41,394 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:41,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35416 deadline: 1732648661392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:41,395 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to fe76acf732b2,42785,1732648423970 for 'test_row_0' in 66e19d6a9a9885e5b2c786f2e4c45a8e of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4184 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T19:16:41,396 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:41,396 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:41,396 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:41,425 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:41,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35426 deadline: 1732648661421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:41,425 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:41,426 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:41,426 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:41,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35454 deadline: 1732648661422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:41,426 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:41,426 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:41,426 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:41,426 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:41,635 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:41,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35426 deadline: 1732648661632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:41,635 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:41,636 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:41,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35454 deadline: 1732648661632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:41,636 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:41,636 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:41,636 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:41,636 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:41,636 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:41,710 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:41,713 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112696bda8669b7e4edbb0c47ae206e28f9b_66e19d6a9a9885e5b2c786f2e4c45a8e to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112696bda8669b7e4edbb0c47ae206e28f9b_66e19d6a9a9885e5b2c786f2e4c45a8e 2024-11-26T19:16:41,714 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/A/d301166e22d0465eadf5f19317a14cd3, store: [table=TestAcidGuarantees family=A region=66e19d6a9a9885e5b2c786f2e4c45a8e] 2024-11-26T19:16:41,715 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/A/d301166e22d0465eadf5f19317a14cd3 is 175, key is test_row_0/A:col10/1732648601294/Put/seqid=0 2024-11-26T19:16:41,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742411_1587 (size=31255) 2024-11-26T19:16:41,944 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:41,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35426 deadline: 1732648661941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:41,944 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:41,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35454 deadline: 1732648661941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:41,944 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:41,945 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:41,945 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:41,945 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:41,945 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:41,945 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:42,119 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=315, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/A/d301166e22d0465eadf5f19317a14cd3 2024-11-26T19:16:42,126 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/B/af1f3490557443da860fdb0890970875 is 50, key is test_row_0/B:col10/1732648601294/Put/seqid=0 2024-11-26T19:16:42,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742412_1588 (size=12301) 2024-11-26T19:16:42,136 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-26T19:16:42,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-26T19:16:42,382 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T19:16:42,382 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T19:16:42,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees 2024-11-26T19:16:42,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-26T19:16:42,385 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T19:16:42,392 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T19:16:42,392 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T19:16:42,455 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:42,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35426 deadline: 1732648662452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:42,456 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:42,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35454 deadline: 1732648662452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:42,456 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:42,456 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:42,456 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:42,457 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:42,457 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:42,457 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:42,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-26T19:16:42,531 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=315 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/B/af1f3490557443da860fdb0890970875 2024-11-26T19:16:42,543 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/C/0b1d53e991d9421f8cf493f236f829ec is 50, key is test_row_0/C:col10/1732648601294/Put/seqid=0 2024-11-26T19:16:42,545 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-26T19:16:42,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e. 2024-11-26T19:16:42,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e. as already flushing 2024-11-26T19:16:42,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e. 2024-11-26T19:16:42,545 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 66e19d6a9a9885e5b2c786f2e4c45a8e, NAME => 'TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:16:42,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 66e19d6a9a9885e5b2c786f2e4c45a8e, NAME => 'TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:16:42,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66e19d6a9a9885e5b2c786f2e4c45a8e, NAME => 'TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66e19d6a9a9885e5b2c786f2e4c45a8e, NAME => 'TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:16:42,547 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742413_1589 (size=12301) 2024-11-26T19:16:42,699 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-26T19:16:42,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e. 2024-11-26T19:16:42,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e. as already flushing 2024-11-26T19:16:42,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e. 2024-11-26T19:16:42,700 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 66e19d6a9a9885e5b2c786f2e4c45a8e, NAME => 'TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:16:42,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 66e19d6a9a9885e5b2c786f2e4c45a8e, NAME => 'TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:16:42,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66e19d6a9a9885e5b2c786f2e4c45a8e, NAME => 'TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66e19d6a9a9885e5b2c786f2e4c45a8e, NAME => 'TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:16:42,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-26T19:16:42,853 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-26T19:16:42,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e. 2024-11-26T19:16:42,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e. as already flushing 2024-11-26T19:16:42,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e. 2024-11-26T19:16:42,853 ERROR [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 66e19d6a9a9885e5b2c786f2e4c45a8e, NAME => 'TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:16:42,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 66e19d6a9a9885e5b2c786f2e4c45a8e, NAME => 'TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:16:42,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66e19d6a9a9885e5b2c786f2e4c45a8e, NAME => 'TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66e19d6a9a9885e5b2c786f2e4c45a8e, NAME => 'TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:16:42,949 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=315 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/C/0b1d53e991d9421f8cf493f236f829ec 2024-11-26T19:16:42,954 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/A/d301166e22d0465eadf5f19317a14cd3 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/d301166e22d0465eadf5f19317a14cd3 2024-11-26T19:16:42,958 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/d301166e22d0465eadf5f19317a14cd3, entries=150, sequenceid=315, filesize=30.5 K 2024-11-26T19:16:42,959 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/B/af1f3490557443da860fdb0890970875 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/B/af1f3490557443da860fdb0890970875 2024-11-26T19:16:42,963 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/B/af1f3490557443da860fdb0890970875, entries=150, sequenceid=315, filesize=12.0 K 2024-11-26T19:16:42,963 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/C/0b1d53e991d9421f8cf493f236f829ec as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/C/0b1d53e991d9421f8cf493f236f829ec 2024-11-26T19:16:42,967 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/C/0b1d53e991d9421f8cf493f236f829ec, entries=150, sequenceid=315, filesize=12.0 K 2024-11-26T19:16:42,969 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~419.95 KB/430032, currentSize=33.54 KB/34350 for 66e19d6a9a9885e5b2c786f2e4c45a8e in 1674ms, sequenceid=315, compaction requested=true 2024-11-26T19:16:42,969 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 66e19d6a9a9885e5b2c786f2e4c45a8e: 2024-11-26T19:16:42,969 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T19:16:42,970 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 66e19d6a9a9885e5b2c786f2e4c45a8e:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T19:16:42,970 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:16:42,970 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 66e19d6a9a9885e5b2c786f2e4c45a8e:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T19:16:42,970 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:16:42,970 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 66e19d6a9a9885e5b2c786f2e4c45a8e:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T19:16:42,970 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T19:16:42,970 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T19:16:42,971 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T19:16:42,972 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1541): 66e19d6a9a9885e5b2c786f2e4c45a8e/B is initiating minor compaction (all files) 2024-11-26T19:16:42,972 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 66e19d6a9a9885e5b2c786f2e4c45a8e/B in TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e. 2024-11-26T19:16:42,972 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/B/e44547b5e20744f596ab8b11b0bcd794, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/B/6149ecd1bc264563af5527de7993cc86, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/B/af1f3490557443da860fdb0890970875] into tmpdir=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp, totalSize=33.9 K 2024-11-26T19:16:42,972 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 85367 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T19:16:42,972 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1541): 66e19d6a9a9885e5b2c786f2e4c45a8e/A is initiating minor compaction (all files) 2024-11-26T19:16:42,972 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 66e19d6a9a9885e5b2c786f2e4c45a8e/A in TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e. 2024-11-26T19:16:42,972 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/46a2cd1070d14569b1f23ff22d1b6f9d, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/2bf74ae587d042c9a08356c0ebca120f, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/d301166e22d0465eadf5f19317a14cd3] into tmpdir=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp, totalSize=83.4 K 2024-11-26T19:16:42,972 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e. 2024-11-26T19:16:42,972 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e. files: [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/46a2cd1070d14569b1f23ff22d1b6f9d, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/2bf74ae587d042c9a08356c0ebca120f, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/d301166e22d0465eadf5f19317a14cd3] 2024-11-26T19:16:42,973 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting e44547b5e20744f596ab8b11b0bcd794, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1732648597209 2024-11-26T19:16:42,973 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting 46a2cd1070d14569b1f23ff22d1b6f9d, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1732648597209 2024-11-26T19:16:42,973 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting 6149ecd1bc264563af5527de7993cc86, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1732648598335 2024-11-26T19:16:42,973 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2bf74ae587d042c9a08356c0ebca120f, keycount=100, bloomtype=ROW, size=21.9 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1732648598335 2024-11-26T19:16:42,974 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting af1f3490557443da860fdb0890970875, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1732648600650 2024-11-26T19:16:42,974 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting d301166e22d0465eadf5f19317a14cd3, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1732648600650 2024-11-26T19:16:42,983 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=66e19d6a9a9885e5b2c786f2e4c45a8e] 2024-11-26T19:16:42,983 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 66e19d6a9a9885e5b2c786f2e4c45a8e#B#compaction#494 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T19:16:42,984 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/B/030b5955d4cf4e14b3267911ae5935b7 is 50, key is test_row_0/B:col10/1732648601294/Put/seqid=0 2024-11-26T19:16:42,986 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241126f7f748e141574664a614a8dc73ae4188_66e19d6a9a9885e5b2c786f2e4c45a8e store=[table=TestAcidGuarantees family=A region=66e19d6a9a9885e5b2c786f2e4c45a8e] 2024-11-26T19:16:42,988 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241126f7f748e141574664a614a8dc73ae4188_66e19d6a9a9885e5b2c786f2e4c45a8e, store=[table=TestAcidGuarantees family=A region=66e19d6a9a9885e5b2c786f2e4c45a8e] 2024-11-26T19:16:42,988 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126f7f748e141574664a614a8dc73ae4188_66e19d6a9a9885e5b2c786f2e4c45a8e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=66e19d6a9a9885e5b2c786f2e4c45a8e] 2024-11-26T19:16:42,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742414_1590 (size=12949) 2024-11-26T19:16:43,006 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-26T19:16:43,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e. 2024-11-26T19:16:43,006 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2902): Flushing 66e19d6a9a9885e5b2c786f2e4c45a8e 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-26T19:16:43,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66e19d6a9a9885e5b2c786f2e4c45a8e, store=A 2024-11-26T19:16:43,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:16:43,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66e19d6a9a9885e5b2c786f2e4c45a8e, store=B 2024-11-26T19:16:43,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:16:43,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66e19d6a9a9885e5b2c786f2e4c45a8e, store=C 2024-11-26T19:16:43,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:16:43,007 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/B/030b5955d4cf4e14b3267911ae5935b7 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/B/030b5955d4cf4e14b3267911ae5935b7 2024-11-26T19:16:43,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-26T19:16:43,015 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 66e19d6a9a9885e5b2c786f2e4c45a8e/B of 66e19d6a9a9885e5b2c786f2e4c45a8e into 030b5955d4cf4e14b3267911ae5935b7(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T19:16:43,015 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 66e19d6a9a9885e5b2c786f2e4c45a8e: 2024-11-26T19:16:43,015 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., storeName=66e19d6a9a9885e5b2c786f2e4c45a8e/B, priority=13, startTime=1732648602970; duration=0sec 2024-11-26T19:16:43,015 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T19:16:43,015 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 66e19d6a9a9885e5b2c786f2e4c45a8e:B 2024-11-26T19:16:43,015 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T19:16:43,016 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T19:16:43,016 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1541): 66e19d6a9a9885e5b2c786f2e4c45a8e/C is initiating minor compaction (all files) 2024-11-26T19:16:43,016 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 66e19d6a9a9885e5b2c786f2e4c45a8e/C in TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e. 2024-11-26T19:16:43,016 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/C/f6aa479c4752431b8b11bc35a1dd2c7b, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/C/cb3a0b5aca7e4857b9d4ae30b3ed702e, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/C/0b1d53e991d9421f8cf493f236f829ec] into tmpdir=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp, totalSize=33.9 K 2024-11-26T19:16:43,016 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting f6aa479c4752431b8b11bc35a1dd2c7b, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1732648597209 2024-11-26T19:16:43,017 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting cb3a0b5aca7e4857b9d4ae30b3ed702e, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1732648598335 2024-11-26T19:16:43,017 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting 0b1d53e991d9421f8cf493f236f829ec, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1732648600650 2024-11-26T19:16:43,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742415_1591 (size=4469) 2024-11-26T19:16:43,024 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 66e19d6a9a9885e5b2c786f2e4c45a8e#A#compaction#495 average throughput is 0.60 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T19:16:43,025 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/A/43e8185d1b654de29a8603bfc9e7f606 is 175, key is test_row_0/A:col10/1732648601294/Put/seqid=0 2024-11-26T19:16:43,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112652ada8e4a3bd43e5b12a620372f870a6_66e19d6a9a9885e5b2c786f2e4c45a8e is 50, key is test_row_0/A:col10/1732648601309/Put/seqid=0 2024-11-26T19:16:43,038 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 66e19d6a9a9885e5b2c786f2e4c45a8e#C#compaction#497 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T19:16:43,038 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/C/e79c2a3985154b9786f5bc8e45188109 is 50, key is test_row_0/C:col10/1732648601294/Put/seqid=0 2024-11-26T19:16:43,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742416_1592 (size=31903) 2024-11-26T19:16:43,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742418_1594 (size=12949) 2024-11-26T19:16:43,057 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/C/e79c2a3985154b9786f5bc8e45188109 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/C/e79c2a3985154b9786f5bc8e45188109 2024-11-26T19:16:43,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742417_1593 (size=12454) 2024-11-26T19:16:43,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:43,064 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 66e19d6a9a9885e5b2c786f2e4c45a8e/C of 66e19d6a9a9885e5b2c786f2e4c45a8e into e79c2a3985154b9786f5bc8e45188109(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T19:16:43,064 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 66e19d6a9a9885e5b2c786f2e4c45a8e: 2024-11-26T19:16:43,065 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., storeName=66e19d6a9a9885e5b2c786f2e4c45a8e/C, priority=13, startTime=1732648602970; duration=0sec 2024-11-26T19:16:43,065 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:16:43,065 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 66e19d6a9a9885e5b2c786f2e4c45a8e:C 2024-11-26T19:16:43,066 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112652ada8e4a3bd43e5b12a620372f870a6_66e19d6a9a9885e5b2c786f2e4c45a8e to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112652ada8e4a3bd43e5b12a620372f870a6_66e19d6a9a9885e5b2c786f2e4c45a8e 2024-11-26T19:16:43,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/A/f56d672b42624d72a0a98bb4d648917b, store: [table=TestAcidGuarantees family=A region=66e19d6a9a9885e5b2c786f2e4c45a8e] 2024-11-26T19:16:43,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/A/f56d672b42624d72a0a98bb4d648917b is 175, key is test_row_0/A:col10/1732648601309/Put/seqid=0 2024-11-26T19:16:43,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742419_1595 (size=31255) 2024-11-26T19:16:43,455 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/A/43e8185d1b654de29a8603bfc9e7f606 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/43e8185d1b654de29a8603bfc9e7f606 2024-11-26T19:16:43,460 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 66e19d6a9a9885e5b2c786f2e4c45a8e/A of 66e19d6a9a9885e5b2c786f2e4c45a8e into 43e8185d1b654de29a8603bfc9e7f606(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T19:16:43,460 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 66e19d6a9a9885e5b2c786f2e4c45a8e: 2024-11-26T19:16:43,460 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., storeName=66e19d6a9a9885e5b2c786f2e4c45a8e/A, priority=13, startTime=1732648602969; duration=0sec 2024-11-26T19:16:43,460 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:16:43,461 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 66e19d6a9a9885e5b2c786f2e4c45a8e:A 2024-11-26T19:16:43,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(8855): Flush requested on 66e19d6a9a9885e5b2c786f2e4c45a8e 2024-11-26T19:16:43,474 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e. as already flushing 2024-11-26T19:16:43,477 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=323, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/A/f56d672b42624d72a0a98bb4d648917b 2024-11-26T19:16:43,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/B/cc6d07f7b13244b4b1bed0381883286a is 50, key is test_row_0/B:col10/1732648601309/Put/seqid=0 2024-11-26T19:16:43,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742420_1596 (size=12301) 2024-11-26T19:16:43,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-26T19:16:43,525 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732648603499, max timestamp=1732648603521Num uniques 859; 2024-11-26T19:16:43,525 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-26T19:16:43,526 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732648603499, max timestamp=1732648603523Num uniques 859; 2024-11-26T19:16:43,526 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-26T19:16:43,526 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732648603499, max timestamp=1732648603523Num uniques 859; 2024-11-26T19:16:43,526 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-26T19:16:43,641 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732648603597, max timestamp=1732648603637Num uniques 1068; 2024-11-26T19:16:43,642 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-26T19:16:43,642 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732648603597, max timestamp=1732648603637Num uniques 1068; 2024-11-26T19:16:43,642 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-26T19:16:43,645 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732648603597, max timestamp=1732648603637Num uniques 1068; 2024-11-26T19:16:43,645 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-26T19:16:43,749 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1732648603693, max timestamp=1732648603744Num uniques 1027; 2024-11-26T19:16:43,749 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-26T19:16:43,750 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1732648603693, max timestamp=1732648603744Num uniques 1027; 2024-11-26T19:16:43,750 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-26T19:16:43,752 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1732648603693, max timestamp=1732648603744Num uniques 1027; 2024-11-26T19:16:43,752 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-26T19:16:43,797 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8506, totalHeapSize=19584, min timestamp=1732648603783, max timestamp=1732648603792Num uniques 1036; 2024-11-26T19:16:43,797 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47436, new segment data size=8506, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-11-26T19:16:43,798 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8511, totalHeapSize=19584, min timestamp=1732648603783, max timestamp=1732648603792Num uniques 1036; 2024-11-26T19:16:43,798 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47441, new segment data size=8511, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-11-26T19:16:43,801 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1732648603783, max timestamp=1732648603792Num uniques 1036; 2024-11-26T19:16:43,801 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47436, new segment data size=6870, suffix heap size=135712,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=150 2024-11-26T19:16:43,877 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1732648603854, max timestamp=1732648603871Num uniques 1045; 2024-11-26T19:16:43,877 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=11220, suffix heap size=136648,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=245 2024-11-26T19:16:43,877 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1732648603854, max timestamp=1732648603871Num uniques 1009; 2024-11-26T19:16:43,877 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1732648603854, max timestamp=1732648603871Num uniques 1045; 2024-11-26T19:16:43,878 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46224, new segment data size=11220, suffix heap size=132904,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=245 2024-11-26T19:16:43,878 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=11220, suffix heap size=136648,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=245 2024-11-26T19:16:43,904 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=323 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/B/cc6d07f7b13244b4b1bed0381883286a 2024-11-26T19:16:43,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/C/8c91ed7d69f249af88fce89f9c831f2f is 50, key is test_row_0/C:col10/1732648601309/Put/seqid=0 2024-11-26T19:16:43,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742421_1597 (size=12301) 2024-11-26T19:16:43,932 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=323 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/C/8c91ed7d69f249af88fce89f9c831f2f 2024-11-26T19:16:43,934 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:43,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35426 deadline: 1732648663925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:43,935 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:43,935 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:43,935 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:43,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/A/f56d672b42624d72a0a98bb4d648917b as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/f56d672b42624d72a0a98bb4d648917b 2024-11-26T19:16:43,941 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/f56d672b42624d72a0a98bb4d648917b, entries=150, sequenceid=323, filesize=30.5 K 2024-11-26T19:16:43,941 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:43,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35438 deadline: 1732648663933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:43,942 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:43,942 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:43,942 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:43,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35454 deadline: 1732648663935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:43,942 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:43,942 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:43,942 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:43,942 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:43,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/B/cc6d07f7b13244b4b1bed0381883286a as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/B/cc6d07f7b13244b4b1bed0381883286a 2024-11-26T19:16:43,947 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/B/cc6d07f7b13244b4b1bed0381883286a, entries=150, sequenceid=323, filesize=12.0 K 2024-11-26T19:16:43,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/C/8c91ed7d69f249af88fce89f9c831f2f as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/C/8c91ed7d69f249af88fce89f9c831f2f 2024-11-26T19:16:43,960 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/C/8c91ed7d69f249af88fce89f9c831f2f, entries=150, sequenceid=323, filesize=12.0 K 2024-11-26T19:16:43,962 INFO [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=147.60 KB/151140 for 66e19d6a9a9885e5b2c786f2e4c45a8e in 956ms, sequenceid=323, compaction requested=false 2024-11-26T19:16:43,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2603): Flush status journal for 66e19d6a9a9885e5b2c786f2e4c45a8e: 2024-11-26T19:16:43,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e. 2024-11-26T19:16:43,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=159 2024-11-26T19:16:43,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster(4169): Remote procedure done, pid=159 2024-11-26T19:16:43,968 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-11-26T19:16:43,968 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5720 sec 2024-11-26T19:16:43,970 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees in 1.5870 sec 2024-11-26T19:16:44,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(8855): Flush requested on 66e19d6a9a9885e5b2c786f2e4c45a8e 2024-11-26T19:16:44,046 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 66e19d6a9a9885e5b2c786f2e4c45a8e 3/3 column families, dataSize=154.31 KB heapSize=437.39 KB 2024-11-26T19:16:44,046 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66e19d6a9a9885e5b2c786f2e4c45a8e, store=A 2024-11-26T19:16:44,046 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-26T19:16:44,046 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66e19d6a9a9885e5b2c786f2e4c45a8e, store=B 2024-11-26T19:16:44,046 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-26T19:16:44,046 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66e19d6a9a9885e5b2c786f2e4c45a8e, store=C 2024-11-26T19:16:44,046 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-26T19:16:44,056 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126daaf468e98b847adbe393ef951affcba_66e19d6a9a9885e5b2c786f2e4c45a8e is 50, key is test_row_0/A:col10/1732648604044/Put/seqid=0 2024-11-26T19:16:44,063 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:44,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35438 deadline: 1732648664057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:44,063 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:44,063 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:44,063 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:44,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35454 deadline: 1732648664058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:44,063 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:44,064 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:44,064 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:44,064 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:44,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742422_1598 (size=12454) 2024-11-26T19:16:44,066 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:44,070 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126daaf468e98b847adbe393ef951affcba_66e19d6a9a9885e5b2c786f2e4c45a8e to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126daaf468e98b847adbe393ef951affcba_66e19d6a9a9885e5b2c786f2e4c45a8e 2024-11-26T19:16:44,071 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/A/876f2ac06b58413ca60360273c21f9f9, store: [table=TestAcidGuarantees family=A region=66e19d6a9a9885e5b2c786f2e4c45a8e] 2024-11-26T19:16:44,071 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:44,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35426 deadline: 1732648664063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:44,072 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:44,072 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:44,072 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:44,072 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/A/876f2ac06b58413ca60360273c21f9f9 is 175, key is test_row_0/A:col10/1732648604044/Put/seqid=0 2024-11-26T19:16:44,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742423_1599 (size=31255) 2024-11-26T19:16:44,076 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=433, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/A/876f2ac06b58413ca60360273c21f9f9 2024-11-26T19:16:44,094 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/B/baf08bd11b954ecda254b5aec7d5fae3 is 50, key is test_row_0/B:col10/1732648604044/Put/seqid=0 2024-11-26T19:16:44,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742424_1600 (size=12301) 2024-11-26T19:16:44,099 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=433 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/B/baf08bd11b954ecda254b5aec7d5fae3 2024-11-26T19:16:44,105 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/C/ea4b28c72c554ac4ab8c7537a21c58d2 is 50, key is test_row_0/C:col10/1732648604044/Put/seqid=0 2024-11-26T19:16:44,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742425_1601 (size=12301) 2024-11-26T19:16:44,116 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=433 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/C/ea4b28c72c554ac4ab8c7537a21c58d2 2024-11-26T19:16:44,121 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/A/876f2ac06b58413ca60360273c21f9f9 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/876f2ac06b58413ca60360273c21f9f9 2024-11-26T19:16:44,131 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/876f2ac06b58413ca60360273c21f9f9, entries=150, sequenceid=433, filesize=30.5 K 2024-11-26T19:16:44,131 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/B/baf08bd11b954ecda254b5aec7d5fae3 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/B/baf08bd11b954ecda254b5aec7d5fae3 2024-11-26T19:16:44,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:44,138 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/B/baf08bd11b954ecda254b5aec7d5fae3, entries=150, sequenceid=433, filesize=12.0 K 2024-11-26T19:16:44,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:44,140 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/C/ea4b28c72c554ac4ab8c7537a21c58d2 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/C/ea4b28c72c554ac4ab8c7537a21c58d2 2024-11-26T19:16:44,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:44,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:44,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:44,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:44,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:44,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:44,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:44,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:44,145 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/C/ea4b28c72c554ac4ab8c7537a21c58d2, entries=150, sequenceid=433, filesize=12.0 K 2024-11-26T19:16:44,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:44,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:44,146 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~437.34 KB/447840, currentSize=26.84 KB/27480 for 66e19d6a9a9885e5b2c786f2e4c45a8e in 100ms, sequenceid=433, compaction requested=true 2024-11-26T19:16:44,147 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 66e19d6a9a9885e5b2c786f2e4c45a8e: 2024-11-26T19:16:44,147 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 66e19d6a9a9885e5b2c786f2e4c45a8e:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T19:16:44,147 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T19:16:44,147 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:16:44,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:44,147 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T19:16:44,147 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 66e19d6a9a9885e5b2c786f2e4c45a8e:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T19:16:44,147 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:16:44,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:44,148 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 66e19d6a9a9885e5b2c786f2e4c45a8e:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T19:16:44,148 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T19:16:44,148 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T19:16:44,148 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94413 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T19:16:44,148 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1541): 66e19d6a9a9885e5b2c786f2e4c45a8e/A is initiating minor compaction (all files) 2024-11-26T19:16:44,148 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1541): 66e19d6a9a9885e5b2c786f2e4c45a8e/B is initiating minor compaction (all files) 2024-11-26T19:16:44,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:44,148 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 66e19d6a9a9885e5b2c786f2e4c45a8e/B in TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e. 2024-11-26T19:16:44,148 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 66e19d6a9a9885e5b2c786f2e4c45a8e/A in TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e. 2024-11-26T19:16:44,148 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/B/030b5955d4cf4e14b3267911ae5935b7, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/B/cc6d07f7b13244b4b1bed0381883286a, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/B/baf08bd11b954ecda254b5aec7d5fae3] into tmpdir=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp, totalSize=36.7 K 2024-11-26T19:16:44,148 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/43e8185d1b654de29a8603bfc9e7f606, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/f56d672b42624d72a0a98bb4d648917b, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/876f2ac06b58413ca60360273c21f9f9] into tmpdir=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp, totalSize=92.2 K 2024-11-26T19:16:44,148 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e. 2024-11-26T19:16:44,148 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e. files: [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/43e8185d1b654de29a8603bfc9e7f606, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/f56d672b42624d72a0a98bb4d648917b, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/876f2ac06b58413ca60360273c21f9f9] 2024-11-26T19:16:44,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:44,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:44,150 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting 030b5955d4cf4e14b3267911ae5935b7, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1732648600650 2024-11-26T19:16:44,150 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting 43e8185d1b654de29a8603bfc9e7f606, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1732648600650 2024-11-26T19:16:44,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:44,150 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting f56d672b42624d72a0a98bb4d648917b, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1732648601296 2024-11-26T19:16:44,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:44,151 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting 876f2ac06b58413ca60360273c21f9f9, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=433, earliestPutTs=1732648603929 2024-11-26T19:16:44,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:44,151 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting cc6d07f7b13244b4b1bed0381883286a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1732648601296 2024-11-26T19:16:44,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:44,152 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting baf08bd11b954ecda254b5aec7d5fae3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=433, earliestPutTs=1732648603929 2024-11-26T19:16:44,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:44,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:44,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:44,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:44,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:44,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:44,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:44,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:44,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:44,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:44,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:44,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:44,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:44,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:44,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:44,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:44,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:44,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:44,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:44,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:44,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:44,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:44,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:44,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:44,170 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=66e19d6a9a9885e5b2c786f2e4c45a8e] 2024-11-26T19:16:44,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:44,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:44,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:44,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:44,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:44,174 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 66e19d6a9a9885e5b2c786f2e4c45a8e#B#compaction#504 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T19:16:44,175 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/B/bf383d07a7e94268ae36e843a347e1f0 is 50, key is test_row_0/B:col10/1732648604044/Put/seqid=0 2024-11-26T19:16:44,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:44,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:44,185 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241126eaf6b659b7704f55ab981e56729f6a3c_66e19d6a9a9885e5b2c786f2e4c45a8e store=[table=TestAcidGuarantees family=A region=66e19d6a9a9885e5b2c786f2e4c45a8e] 2024-11-26T19:16:44,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:44,185 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 66e19d6a9a9885e5b2c786f2e4c45a8e 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-26T19:16:44,186 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66e19d6a9a9885e5b2c786f2e4c45a8e, store=A 2024-11-26T19:16:44,186 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:16:44,186 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66e19d6a9a9885e5b2c786f2e4c45a8e, store=B 2024-11-26T19:16:44,186 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:16:44,186 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66e19d6a9a9885e5b2c786f2e4c45a8e, store=C 2024-11-26T19:16:44,186 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:16:44,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:44,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(8855): Flush requested on 66e19d6a9a9885e5b2c786f2e4c45a8e 2024-11-26T19:16:44,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:44,187 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241126eaf6b659b7704f55ab981e56729f6a3c_66e19d6a9a9885e5b2c786f2e4c45a8e, store=[table=TestAcidGuarantees family=A region=66e19d6a9a9885e5b2c786f2e4c45a8e] 2024-11-26T19:16:44,187 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126eaf6b659b7704f55ab981e56729f6a3c_66e19d6a9a9885e5b2c786f2e4c45a8e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=66e19d6a9a9885e5b2c786f2e4c45a8e] 2024-11-26T19:16:44,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:44,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:44,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:44,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:44,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:44,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:44,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:44,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:44,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:44,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:44,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:44,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:44,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:44,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:44,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:44,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:44,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:44,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:44,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:44,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:44,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:44,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:44,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:44,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:44,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:44,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:44,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:44,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:44,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:44,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:44,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742426_1602 (size=13051) 2024-11-26T19:16:44,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:44,218 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126197a014aed294ca6a7c1648a090a3884_66e19d6a9a9885e5b2c786f2e4c45a8e is 50, key is test_row_0/A:col10/1732648604180/Put/seqid=0 2024-11-26T19:16:44,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:44,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:44,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:44,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:44,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:44,223 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/B/bf383d07a7e94268ae36e843a347e1f0 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/B/bf383d07a7e94268ae36e843a347e1f0 2024-11-26T19:16:44,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:44,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:44,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:44,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:44,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:44,228 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 66e19d6a9a9885e5b2c786f2e4c45a8e/B of 66e19d6a9a9885e5b2c786f2e4c45a8e into bf383d07a7e94268ae36e843a347e1f0(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T19:16:44,228 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 66e19d6a9a9885e5b2c786f2e4c45a8e: 2024-11-26T19:16:44,228 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., storeName=66e19d6a9a9885e5b2c786f2e4c45a8e/B, priority=13, startTime=1732648604147; duration=0sec 2024-11-26T19:16:44,228 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T19:16:44,228 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 66e19d6a9a9885e5b2c786f2e4c45a8e:B 2024-11-26T19:16:44,228 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T19:16:44,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:44,229 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T19:16:44,229 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1541): 66e19d6a9a9885e5b2c786f2e4c45a8e/C is initiating minor compaction (all files) 2024-11-26T19:16:44,229 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 66e19d6a9a9885e5b2c786f2e4c45a8e/C in TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e. 2024-11-26T19:16:44,230 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/C/e79c2a3985154b9786f5bc8e45188109, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/C/8c91ed7d69f249af88fce89f9c831f2f, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/C/ea4b28c72c554ac4ab8c7537a21c58d2] into tmpdir=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp, totalSize=36.7 K 2024-11-26T19:16:44,230 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting e79c2a3985154b9786f5bc8e45188109, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1732648600650 2024-11-26T19:16:44,230 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting 8c91ed7d69f249af88fce89f9c831f2f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1732648601296 2024-11-26T19:16:44,231 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting ea4b28c72c554ac4ab8c7537a21c58d2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=433, earliestPutTs=1732648603929 2024-11-26T19:16:44,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742427_1603 (size=4469) 2024-11-26T19:16:44,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:44,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:44,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:44,259 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1732648604222, max timestamp=1732648604255Num uniques 859; 2024-11-26T19:16:44,260 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-26T19:16:44,259 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1732648604222, max timestamp=1732648604255Num uniques 859; 2024-11-26T19:16:44,261 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-26T19:16:44,261 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 66e19d6a9a9885e5b2c786f2e4c45a8e#C#compaction#506 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T19:16:44,261 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/C/3de36818353c47d7a012c902b2451f22 is 50, key is test_row_0/C:col10/1732648604044/Put/seqid=0 2024-11-26T19:16:44,262 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732648604222, max timestamp=1732648604255Num uniques 859; 2024-11-26T19:16:44,262 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fe76acf732b2:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-26T19:16:44,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742428_1604 (size=14994) 2024-11-26T19:16:44,306 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:44,309 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:44,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35454 deadline: 1732648664301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:44,309 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:44,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35426 deadline: 1732648664302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:44,310 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:44,310 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:44,310 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:44,310 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:44,310 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:44,310 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:44,311 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126197a014aed294ca6a7c1648a090a3884_66e19d6a9a9885e5b2c786f2e4c45a8e to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126197a014aed294ca6a7c1648a090a3884_66e19d6a9a9885e5b2c786f2e4c45a8e 2024-11-26T19:16:44,312 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/A/1a802674f0544c2eba1b46d3c0267bfb, store: [table=TestAcidGuarantees family=A region=66e19d6a9a9885e5b2c786f2e4c45a8e] 2024-11-26T19:16:44,313 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/A/1a802674f0544c2eba1b46d3c0267bfb is 175, key is test_row_0/A:col10/1732648604180/Put/seqid=0 2024-11-26T19:16:44,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742429_1605 (size=13051) 2024-11-26T19:16:44,315 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:44,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35438 deadline: 1732648664308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:44,315 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:44,315 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:44,315 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:44,320 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/C/3de36818353c47d7a012c902b2451f22 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/C/3de36818353c47d7a012c902b2451f22 2024-11-26T19:16:44,324 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 66e19d6a9a9885e5b2c786f2e4c45a8e/C of 66e19d6a9a9885e5b2c786f2e4c45a8e into 3de36818353c47d7a012c902b2451f22(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T19:16:44,324 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 66e19d6a9a9885e5b2c786f2e4c45a8e: 2024-11-26T19:16:44,324 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., storeName=66e19d6a9a9885e5b2c786f2e4c45a8e/C, priority=13, startTime=1732648604147; duration=0sec 2024-11-26T19:16:44,324 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:16:44,324 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 66e19d6a9a9885e5b2c786f2e4c45a8e:C 2024-11-26T19:16:44,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742430_1606 (size=39949) 2024-11-26T19:16:44,415 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:44,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35454 deadline: 1732648664412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:44,416 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:44,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35426 deadline: 1732648664412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:44,416 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:44,416 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:44,416 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:44,416 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:44,416 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:44,416 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:44,426 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:44,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35438 deadline: 1732648664422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:44,427 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:44,427 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:44,427 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:44,448 INFO [Thread-2315 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2315. 2024-11-26T19:16:44,448 DEBUG [Thread-2315 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T19:16:44,448 DEBUG [Thread-2315 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:16:44,448 DEBUG [Thread-2315 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:16:44,448 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T19:16:44,449 INFO [Thread-2317 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2317. 2024-11-26T19:16:44,449 DEBUG [Thread-2317 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T19:16:44,449 DEBUG [Thread-2317 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:16:44,449 INFO [Thread-2313 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2313. 2024-11-26T19:16:44,449 DEBUG [Thread-2317 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:16:44,449 DEBUG [Thread-2313 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T19:16:44,449 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T19:16:44,449 DEBUG [Thread-2313 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:16:44,449 DEBUG [Thread-2313 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:16:44,449 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T19:16:44,453 INFO [Thread-2321 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2321. 2024-11-26T19:16:44,453 DEBUG [Thread-2321 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T19:16:44,453 DEBUG [Thread-2321 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:16:44,453 DEBUG [Thread-2321 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:16:44,453 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T19:16:44,453 INFO [Thread-2319 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2319. 2024-11-26T19:16:44,453 DEBUG [Thread-2319 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T19:16:44,453 DEBUG [Thread-2319 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:16:44,453 DEBUG [Thread-2319 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:16:44,453 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T19:16:44,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-26T19:16:44,532 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T19:16:44,622 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:44,622 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:44,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35454 deadline: 1732648664622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:44,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35426 deadline: 1732648664622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:44,622 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:44,622 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:44,622 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:44,622 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:44,622 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:44,622 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:44,632 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:44,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35438 deadline: 1732648664632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:44,633 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:44,633 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:44,633 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:44,641 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 66e19d6a9a9885e5b2c786f2e4c45a8e#A#compaction#503 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T19:16:44,642 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/A/429fd324533341e19f682ec8b8298251 is 175, key is test_row_0/A:col10/1732648604044/Put/seqid=0 2024-11-26T19:16:44,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742431_1607 (size=32005) 2024-11-26T19:16:44,734 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=443, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/A/1a802674f0544c2eba1b46d3c0267bfb 2024-11-26T19:16:44,739 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/B/ca886842bcb34290a9412979e3398949 is 50, key is test_row_0/B:col10/1732648604180/Put/seqid=0 2024-11-26T19:16:44,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742432_1608 (size=12301) 2024-11-26T19:16:44,932 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:44,932 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:44,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35454 deadline: 1732648664932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:44,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35426 deadline: 1732648664932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:44,933 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:44,933 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:44,933 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:44,933 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:44,933 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:44,933 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:44,941 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:44,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35438 deadline: 1732648664941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:44,942 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:44,942 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:44,942 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:45,048 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/A/429fd324533341e19f682ec8b8298251 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/429fd324533341e19f682ec8b8298251 2024-11-26T19:16:45,051 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 66e19d6a9a9885e5b2c786f2e4c45a8e/A of 66e19d6a9a9885e5b2c786f2e4c45a8e into 429fd324533341e19f682ec8b8298251(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T19:16:45,051 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 66e19d6a9a9885e5b2c786f2e4c45a8e: 2024-11-26T19:16:45,052 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., storeName=66e19d6a9a9885e5b2c786f2e4c45a8e/A, priority=13, startTime=1732648604147; duration=0sec 2024-11-26T19:16:45,052 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:16:45,052 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 66e19d6a9a9885e5b2c786f2e4c45a8e:A 2024-11-26T19:16:45,143 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=443 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/B/ca886842bcb34290a9412979e3398949 2024-11-26T19:16:45,148 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/C/46fd39636f8743bbbfe3e3431461a1c3 is 50, key is test_row_0/C:col10/1732648604180/Put/seqid=0 2024-11-26T19:16:45,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742433_1609 (size=12301) 2024-11-26T19:16:45,422 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:45,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35456 deadline: 1732648665421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:45,422 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to fe76acf732b2,42785,1732648423970 for 'test_row_2' in 66e19d6a9a9885e5b2c786f2e4c45a8e of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8214 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T19:16:45,423 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:45,423 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:45,423 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:45,442 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:45,442 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:45,442 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:45,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35454 deadline: 1732648665442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:45,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35416 deadline: 1732648665442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:45,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35426 deadline: 1732648665442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:45,442 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:45,442 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:45,443 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:45,443 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:45,443 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:45,443 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:45,442 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to fe76acf732b2,42785,1732648423970 for 'test_row_0' in 66e19d6a9a9885e5b2c786f2e4c45a8e of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8231 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T19:16:45,443 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:45,443 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:45,443 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:45,452 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:16:45,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35438 deadline: 1732648665452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:45,453 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 , the old value is region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:45,453 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T19:16:45,453 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., hostname=fe76acf732b2,42785,1732648423970, seqNum=5 because the exception is null or not the one we care about 2024-11-26T19:16:45,551 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=443 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/C/46fd39636f8743bbbfe3e3431461a1c3 2024-11-26T19:16:45,555 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/A/1a802674f0544c2eba1b46d3c0267bfb as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/1a802674f0544c2eba1b46d3c0267bfb 2024-11-26T19:16:45,557 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/1a802674f0544c2eba1b46d3c0267bfb, entries=200, sequenceid=443, filesize=39.0 K 2024-11-26T19:16:45,558 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/B/ca886842bcb34290a9412979e3398949 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/B/ca886842bcb34290a9412979e3398949 2024-11-26T19:16:45,560 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/B/ca886842bcb34290a9412979e3398949, entries=150, sequenceid=443, filesize=12.0 K 2024-11-26T19:16:45,561 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/C/46fd39636f8743bbbfe3e3431461a1c3 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/C/46fd39636f8743bbbfe3e3431461a1c3 2024-11-26T19:16:45,563 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/C/46fd39636f8743bbbfe3e3431461a1c3, entries=150, sequenceid=443, filesize=12.0 K 2024-11-26T19:16:45,564 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 66e19d6a9a9885e5b2c786f2e4c45a8e in 1379ms, sequenceid=443, compaction requested=false 2024-11-26T19:16:45,564 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 66e19d6a9a9885e5b2c786f2e4c45a8e: 2024-11-26T19:16:46,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42785 {}] regionserver.HRegion(8855): Flush requested on 66e19d6a9a9885e5b2c786f2e4c45a8e 2024-11-26T19:16:46,453 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 66e19d6a9a9885e5b2c786f2e4c45a8e 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-11-26T19:16:46,453 INFO [Thread-2306 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2306. 2024-11-26T19:16:46,453 DEBUG [Thread-2306 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T19:16:46,453 DEBUG [Thread-2306 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:16:46,453 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66e19d6a9a9885e5b2c786f2e4c45a8e, store=A 2024-11-26T19:16:46,453 DEBUG [Thread-2306 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:16:46,453 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T19:16:46,453 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T19:16:46,453 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66e19d6a9a9885e5b2c786f2e4c45a8e, store=B 2024-11-26T19:16:46,453 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T19:16:46,453 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66e19d6a9a9885e5b2c786f2e4c45a8e, store=C 2024-11-26T19:16:46,453 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T19:16:46,458 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126e9cb0cd629fb43b69cca0526eb5b1f52_66e19d6a9a9885e5b2c786f2e4c45a8e is 50, key is test_row_0/A:col10/1732648606452/Put/seqid=0 2024-11-26T19:16:46,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742434_1610 (size=12454) 2024-11-26T19:16:46,463 INFO [Thread-2302 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2302. 2024-11-26T19:16:46,463 DEBUG [Thread-2302 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T19:16:46,463 DEBUG [Thread-2302 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:16:46,463 DEBUG [Thread-2302 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:16:46,463 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T19:16:46,463 INFO [Thread-2310 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2310. 2024-11-26T19:16:46,463 DEBUG [Thread-2310 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T19:16:46,463 DEBUG [Thread-2310 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:16:46,463 DEBUG [Thread-2310 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:16:46,463 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T19:16:46,861 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:46,864 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126e9cb0cd629fb43b69cca0526eb5b1f52_66e19d6a9a9885e5b2c786f2e4c45a8e to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126e9cb0cd629fb43b69cca0526eb5b1f52_66e19d6a9a9885e5b2c786f2e4c45a8e 2024-11-26T19:16:46,865 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/A/a6045f6ff2174af3bdfed1277c848412, store: [table=TestAcidGuarantees family=A region=66e19d6a9a9885e5b2c786f2e4c45a8e] 2024-11-26T19:16:46,865 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/A/a6045f6ff2174af3bdfed1277c848412 is 175, key is test_row_0/A:col10/1732648606452/Put/seqid=0 2024-11-26T19:16:46,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742435_1611 (size=31255) 2024-11-26T19:16:47,269 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=483, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/A/a6045f6ff2174af3bdfed1277c848412 2024-11-26T19:16:47,274 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/B/c37ba3e81fa647b68e17543efdc9bbd7 is 50, key is test_row_0/B:col10/1732648606452/Put/seqid=0 2024-11-26T19:16:47,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742436_1612 (size=12301) 2024-11-26T19:16:47,678 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=483 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/B/c37ba3e81fa647b68e17543efdc9bbd7 2024-11-26T19:16:47,683 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/C/36ee3db87ec148d5ac65a9480b0dc44f is 50, key is test_row_0/C:col10/1732648606452/Put/seqid=0 2024-11-26T19:16:47,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742437_1613 (size=12301) 2024-11-26T19:16:48,086 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=483 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/C/36ee3db87ec148d5ac65a9480b0dc44f 2024-11-26T19:16:48,090 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/A/a6045f6ff2174af3bdfed1277c848412 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/a6045f6ff2174af3bdfed1277c848412 2024-11-26T19:16:48,092 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/a6045f6ff2174af3bdfed1277c848412, entries=150, sequenceid=483, filesize=30.5 K 2024-11-26T19:16:48,093 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/B/c37ba3e81fa647b68e17543efdc9bbd7 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/B/c37ba3e81fa647b68e17543efdc9bbd7 2024-11-26T19:16:48,096 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/B/c37ba3e81fa647b68e17543efdc9bbd7, entries=150, sequenceid=483, filesize=12.0 K 2024-11-26T19:16:48,096 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/C/36ee3db87ec148d5ac65a9480b0dc44f as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/C/36ee3db87ec148d5ac65a9480b0dc44f 2024-11-26T19:16:48,099 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/C/36ee3db87ec148d5ac65a9480b0dc44f, entries=150, sequenceid=483, filesize=12.0 K 2024-11-26T19:16:48,099 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=13.42 KB/13740 for 66e19d6a9a9885e5b2c786f2e4c45a8e in 1646ms, sequenceid=483, compaction requested=true 2024-11-26T19:16:48,099 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 66e19d6a9a9885e5b2c786f2e4c45a8e: 2024-11-26T19:16:48,099 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 66e19d6a9a9885e5b2c786f2e4c45a8e:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T19:16:48,100 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:16:48,100 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 66e19d6a9a9885e5b2c786f2e4c45a8e:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T19:16:48,100 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T19:16:48,100 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:16:48,100 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 66e19d6a9a9885e5b2c786f2e4c45a8e:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T19:16:48,100 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T19:16:48,100 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T19:16:48,100 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T19:16:48,100 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T19:16:48,100 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1541): 66e19d6a9a9885e5b2c786f2e4c45a8e/A is initiating minor compaction (all files) 2024-11-26T19:16:48,100 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1541): 66e19d6a9a9885e5b2c786f2e4c45a8e/B is initiating minor compaction (all files) 2024-11-26T19:16:48,100 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 66e19d6a9a9885e5b2c786f2e4c45a8e/A in TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e. 2024-11-26T19:16:48,100 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 66e19d6a9a9885e5b2c786f2e4c45a8e/B in TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e. 2024-11-26T19:16:48,100 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/429fd324533341e19f682ec8b8298251, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/1a802674f0544c2eba1b46d3c0267bfb, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/a6045f6ff2174af3bdfed1277c848412] into tmpdir=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp, totalSize=100.8 K 2024-11-26T19:16:48,100 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/B/bf383d07a7e94268ae36e843a347e1f0, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/B/ca886842bcb34290a9412979e3398949, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/B/c37ba3e81fa647b68e17543efdc9bbd7] into tmpdir=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp, totalSize=36.8 K 2024-11-26T19:16:48,100 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e. 2024-11-26T19:16:48,100 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e. files: [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/429fd324533341e19f682ec8b8298251, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/1a802674f0544c2eba1b46d3c0267bfb, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/a6045f6ff2174af3bdfed1277c848412] 2024-11-26T19:16:48,101 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting bf383d07a7e94268ae36e843a347e1f0, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=433, earliestPutTs=1732648603929 2024-11-26T19:16:48,101 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting 429fd324533341e19f682ec8b8298251, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=433, earliestPutTs=1732648603929 2024-11-26T19:16:48,101 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting ca886842bcb34290a9412979e3398949, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=443, earliestPutTs=1732648604177 2024-11-26T19:16:48,101 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1a802674f0544c2eba1b46d3c0267bfb, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=443, earliestPutTs=1732648604053 2024-11-26T19:16:48,101 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting c37ba3e81fa647b68e17543efdc9bbd7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=483, earliestPutTs=1732648604296 2024-11-26T19:16:48,101 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] compactions.Compactor(225): Compacting a6045f6ff2174af3bdfed1277c848412, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=483, earliestPutTs=1732648604296 2024-11-26T19:16:48,107 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=66e19d6a9a9885e5b2c786f2e4c45a8e] 2024-11-26T19:16:48,108 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112643d73b2d246743a885876b66454bf8bc_66e19d6a9a9885e5b2c786f2e4c45a8e store=[table=TestAcidGuarantees family=A region=66e19d6a9a9885e5b2c786f2e4c45a8e] 2024-11-26T19:16:48,108 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 66e19d6a9a9885e5b2c786f2e4c45a8e#B#compaction#513 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T19:16:48,109 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/B/99778df0ed484c2687699f40e64372db is 50, key is test_row_0/B:col10/1732648606452/Put/seqid=0 2024-11-26T19:16:48,110 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112643d73b2d246743a885876b66454bf8bc_66e19d6a9a9885e5b2c786f2e4c45a8e, store=[table=TestAcidGuarantees family=A region=66e19d6a9a9885e5b2c786f2e4c45a8e] 2024-11-26T19:16:48,111 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112643d73b2d246743a885876b66454bf8bc_66e19d6a9a9885e5b2c786f2e4c45a8e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=66e19d6a9a9885e5b2c786f2e4c45a8e] 2024-11-26T19:16:48,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742438_1614 (size=13153) 2024-11-26T19:16:48,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742439_1615 (size=4469) 2024-11-26T19:16:48,517 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/B/99778df0ed484c2687699f40e64372db as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/B/99778df0ed484c2687699f40e64372db 2024-11-26T19:16:48,518 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 66e19d6a9a9885e5b2c786f2e4c45a8e#A#compaction#512 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T19:16:48,519 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/A/042390db0c76490892d41a81067a0f68 is 175, key is test_row_0/A:col10/1732648606452/Put/seqid=0 2024-11-26T19:16:48,521 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 66e19d6a9a9885e5b2c786f2e4c45a8e/B of 66e19d6a9a9885e5b2c786f2e4c45a8e into 99778df0ed484c2687699f40e64372db(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T19:16:48,521 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 66e19d6a9a9885e5b2c786f2e4c45a8e: 2024-11-26T19:16:48,521 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., storeName=66e19d6a9a9885e5b2c786f2e4c45a8e/B, priority=13, startTime=1732648608100; duration=0sec 2024-11-26T19:16:48,521 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T19:16:48,521 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 66e19d6a9a9885e5b2c786f2e4c45a8e:B 2024-11-26T19:16:48,521 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T19:16:48,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742440_1616 (size=32107) 2024-11-26T19:16:48,522 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T19:16:48,522 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1541): 66e19d6a9a9885e5b2c786f2e4c45a8e/C is initiating minor compaction (all files) 2024-11-26T19:16:48,522 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 66e19d6a9a9885e5b2c786f2e4c45a8e/C in TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e. 2024-11-26T19:16:48,522 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/C/3de36818353c47d7a012c902b2451f22, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/C/46fd39636f8743bbbfe3e3431461a1c3, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/C/36ee3db87ec148d5ac65a9480b0dc44f] into tmpdir=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp, totalSize=36.8 K 2024-11-26T19:16:48,522 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting 3de36818353c47d7a012c902b2451f22, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=433, earliestPutTs=1732648603929 2024-11-26T19:16:48,522 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting 46fd39636f8743bbbfe3e3431461a1c3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=443, earliestPutTs=1732648604177 2024-11-26T19:16:48,523 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] compactions.Compactor(225): Compacting 36ee3db87ec148d5ac65a9480b0dc44f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=483, earliestPutTs=1732648604296 2024-11-26T19:16:48,528 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 66e19d6a9a9885e5b2c786f2e4c45a8e#C#compaction#514 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T19:16:48,528 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/C/c06ced022c4b496fac6fdb7e415b60e5 is 50, key is test_row_0/C:col10/1732648606452/Put/seqid=0 2024-11-26T19:16:48,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742441_1617 (size=13153) 2024-11-26T19:16:48,925 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/A/042390db0c76490892d41a81067a0f68 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/042390db0c76490892d41a81067a0f68 2024-11-26T19:16:48,928 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 66e19d6a9a9885e5b2c786f2e4c45a8e/A of 66e19d6a9a9885e5b2c786f2e4c45a8e into 042390db0c76490892d41a81067a0f68(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T19:16:48,928 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 66e19d6a9a9885e5b2c786f2e4c45a8e: 2024-11-26T19:16:48,929 INFO [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., storeName=66e19d6a9a9885e5b2c786f2e4c45a8e/A, priority=13, startTime=1732648608099; duration=0sec 2024-11-26T19:16:48,929 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:16:48,929 DEBUG [RS:0;fe76acf732b2:42785-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 66e19d6a9a9885e5b2c786f2e4c45a8e:A 2024-11-26T19:16:48,934 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/C/c06ced022c4b496fac6fdb7e415b60e5 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/C/c06ced022c4b496fac6fdb7e415b60e5 2024-11-26T19:16:48,938 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 66e19d6a9a9885e5b2c786f2e4c45a8e/C of 66e19d6a9a9885e5b2c786f2e4c45a8e into c06ced022c4b496fac6fdb7e415b60e5(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T19:16:48,938 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 66e19d6a9a9885e5b2c786f2e4c45a8e: 2024-11-26T19:16:48,938 INFO [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., storeName=66e19d6a9a9885e5b2c786f2e4c45a8e/C, priority=13, startTime=1732648608100; duration=0sec 2024-11-26T19:16:48,938 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T19:16:48,938 DEBUG [RS:0;fe76acf732b2:42785-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 66e19d6a9a9885e5b2c786f2e4c45a8e:C 2024-11-26T19:16:55,534 INFO [Thread-2308 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2308. 2024-11-26T19:16:55,534 INFO [Thread-2304 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2304. 2024-11-26T19:16:55,534 DEBUG [Thread-2308 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T19:16:55,534 DEBUG [Thread-2304 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T19:16:55,534 DEBUG [Thread-2308 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:16:55,534 DEBUG [Thread-2304 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:16:55,534 DEBUG [Thread-2308 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:16:55,534 DEBUG [Thread-2304 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:16:55,534 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T19:16:55,534 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T19:16:55,534 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-26T19:16:55,534 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 72 2024-11-26T19:16:55,534 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 45 2024-11-26T19:16:55,534 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 125 2024-11-26T19:16:55,534 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 47 2024-11-26T19:16:55,534 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 119 2024-11-26T19:16:55,534 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-26T19:16:55,534 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2885 2024-11-26T19:16:55,534 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2988 2024-11-26T19:16:55,534 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2851 2024-11-26T19:16:55,534 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2894 2024-11-26T19:16:55,534 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2983 2024-11-26T19:16:55,534 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-26T19:16:55,534 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-26T19:16:55,535 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-26T19:16:55,535 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:16:55,535 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:16:55,535 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T19:16:55,535 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-26T19:16:55,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-26T19:16:55,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-26T19:16:55,538 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732648615538"}]},"ts":"1732648615538"} 2024-11-26T19:16:55,539 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-26T19:16:55,539 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-26T19:16:55,540 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-26T19:16:55,541 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=162, ppid=161, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=66e19d6a9a9885e5b2c786f2e4c45a8e, UNASSIGN}] 2024-11-26T19:16:55,541 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=162, ppid=161, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=66e19d6a9a9885e5b2c786f2e4c45a8e, UNASSIGN 2024-11-26T19:16:55,542 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=162 updating hbase:meta row=66e19d6a9a9885e5b2c786f2e4c45a8e, regionState=CLOSING, regionLocation=fe76acf732b2,42785,1732648423970 2024-11-26T19:16:55,543 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=162, ppid=161, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=66e19d6a9a9885e5b2c786f2e4c45a8e, UNASSIGN because future has completed 2024-11-26T19:16:55,543 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-26T19:16:55,543 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE, hasLock=false; CloseRegionProcedure 66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970}] 2024-11-26T19:16:55,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-26T19:16:55,695 INFO [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] handler.UnassignRegionHandler(122): Close 66e19d6a9a9885e5b2c786f2e4c45a8e 2024-11-26T19:16:55,695 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-26T19:16:55,695 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegion(1722): Closing 66e19d6a9a9885e5b2c786f2e4c45a8e, disabling compactions & flushes 2024-11-26T19:16:55,695 INFO [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e. 2024-11-26T19:16:55,696 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e. 2024-11-26T19:16:55,696 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e. after waiting 0 ms 2024-11-26T19:16:55,696 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e. 2024-11-26T19:16:55,696 INFO [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegion(2902): Flushing 66e19d6a9a9885e5b2c786f2e4c45a8e 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-26T19:16:55,696 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66e19d6a9a9885e5b2c786f2e4c45a8e, store=A 2024-11-26T19:16:55,696 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:16:55,696 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66e19d6a9a9885e5b2c786f2e4c45a8e, store=B 2024-11-26T19:16:55,696 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:16:55,696 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66e19d6a9a9885e5b2c786f2e4c45a8e, store=C 2024-11-26T19:16:55,696 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T19:16:55,701 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411260454914cdd1241479e8214a573632974_66e19d6a9a9885e5b2c786f2e4c45a8e is 50, key is test_row_0/A:col10/1732648615532/Put/seqid=0 2024-11-26T19:16:55,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742442_1618 (size=12454) 2024-11-26T19:16:55,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-26T19:16:56,104 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:16:56,107 INFO [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411260454914cdd1241479e8214a573632974_66e19d6a9a9885e5b2c786f2e4c45a8e to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411260454914cdd1241479e8214a573632974_66e19d6a9a9885e5b2c786f2e4c45a8e 2024-11-26T19:16:56,108 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/A/5afaa5fdaaa645fca95b64fd46693212, store: [table=TestAcidGuarantees family=A region=66e19d6a9a9885e5b2c786f2e4c45a8e] 2024-11-26T19:16:56,108 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/A/5afaa5fdaaa645fca95b64fd46693212 is 175, key is test_row_0/A:col10/1732648615532/Put/seqid=0 2024-11-26T19:16:56,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742443_1619 (size=31255) 2024-11-26T19:16:56,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-26T19:16:56,512 INFO [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=493, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/A/5afaa5fdaaa645fca95b64fd46693212 2024-11-26T19:16:56,517 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/B/4f4c59ff67114bb992ce7ee308c008e8 is 50, key is test_row_0/B:col10/1732648615532/Put/seqid=0 2024-11-26T19:16:56,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742444_1620 (size=12301) 2024-11-26T19:16:56,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-26T19:16:56,921 INFO [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=493 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/B/4f4c59ff67114bb992ce7ee308c008e8 2024-11-26T19:16:56,926 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/C/4f4e0cb76c9744ceae642d574c1ab9d6 is 50, key is test_row_0/C:col10/1732648615532/Put/seqid=0 2024-11-26T19:16:56,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742445_1621 (size=12301) 2024-11-26T19:16:57,329 INFO [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=493 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/C/4f4e0cb76c9744ceae642d574c1ab9d6 2024-11-26T19:16:57,332 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/A/5afaa5fdaaa645fca95b64fd46693212 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/5afaa5fdaaa645fca95b64fd46693212 2024-11-26T19:16:57,335 INFO [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/5afaa5fdaaa645fca95b64fd46693212, entries=150, sequenceid=493, filesize=30.5 K 2024-11-26T19:16:57,335 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/B/4f4c59ff67114bb992ce7ee308c008e8 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/B/4f4c59ff67114bb992ce7ee308c008e8 2024-11-26T19:16:57,338 INFO [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/B/4f4c59ff67114bb992ce7ee308c008e8, entries=150, sequenceid=493, filesize=12.0 K 2024-11-26T19:16:57,338 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/.tmp/C/4f4e0cb76c9744ceae642d574c1ab9d6 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/C/4f4e0cb76c9744ceae642d574c1ab9d6 2024-11-26T19:16:57,340 INFO [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/C/4f4e0cb76c9744ceae642d574c1ab9d6, entries=150, sequenceid=493, filesize=12.0 K 2024-11-26T19:16:57,341 INFO [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 66e19d6a9a9885e5b2c786f2e4c45a8e in 1645ms, sequenceid=493, compaction requested=false 2024-11-26T19:16:57,341 DEBUG [StoreCloser-TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/87a642e497174afca90efe8d898f0b53, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/b157fb2cef044d1898092359ba36acc0, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/6f49ed068f664ec4986f5d0d4ed468dd, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/1348ef0ddc8e48d1a99546dee9f1b03a, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/a02ced0c15b64db7a036f0c9099f4a51, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/c1d35cb2eff54c858e5f6ec86ffdc19e, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/424faf27119a487bae4296921a3dc5f4, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/fc89df7a6f5c497bb05ae88db2ac6aeb, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/549a987d77de492d8546f7ef8da171db, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/bb50fc14458e4ff493f25015fe5a5e0f, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/52297031376c4960b8d295fc2485d9c0, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/d9629b526b2840e29e5f097c0c495118, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/bf7e8f9b4a7048d0b9cdfd192ee0f77a, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/0ddccb6d4dfc4b54b2c508ce52bf5713, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/ee38a39e45ca41c4ba673e6ea07bbb38, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/076c35a352c64be0a3cda5c12aa5731b, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/46a2cd1070d14569b1f23ff22d1b6f9d, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/2bf74ae587d042c9a08356c0ebca120f, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/43e8185d1b654de29a8603bfc9e7f606, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/d301166e22d0465eadf5f19317a14cd3, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/f56d672b42624d72a0a98bb4d648917b, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/429fd324533341e19f682ec8b8298251, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/876f2ac06b58413ca60360273c21f9f9, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/1a802674f0544c2eba1b46d3c0267bfb, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/a6045f6ff2174af3bdfed1277c848412] to archive 2024-11-26T19:16:57,342 DEBUG [StoreCloser-TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-26T19:16:57,343 DEBUG [StoreCloser-TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/87a642e497174afca90efe8d898f0b53 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/87a642e497174afca90efe8d898f0b53 2024-11-26T19:16:57,344 DEBUG [StoreCloser-TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/b157fb2cef044d1898092359ba36acc0 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/b157fb2cef044d1898092359ba36acc0 2024-11-26T19:16:57,345 DEBUG [StoreCloser-TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/6f49ed068f664ec4986f5d0d4ed468dd to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/6f49ed068f664ec4986f5d0d4ed468dd 2024-11-26T19:16:57,345 DEBUG [StoreCloser-TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/1348ef0ddc8e48d1a99546dee9f1b03a to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/1348ef0ddc8e48d1a99546dee9f1b03a 2024-11-26T19:16:57,346 DEBUG [StoreCloser-TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/a02ced0c15b64db7a036f0c9099f4a51 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/a02ced0c15b64db7a036f0c9099f4a51 2024-11-26T19:16:57,347 DEBUG [StoreCloser-TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/c1d35cb2eff54c858e5f6ec86ffdc19e to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/c1d35cb2eff54c858e5f6ec86ffdc19e 2024-11-26T19:16:57,348 DEBUG [StoreCloser-TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/424faf27119a487bae4296921a3dc5f4 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/424faf27119a487bae4296921a3dc5f4 2024-11-26T19:16:57,348 DEBUG [StoreCloser-TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/fc89df7a6f5c497bb05ae88db2ac6aeb to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/fc89df7a6f5c497bb05ae88db2ac6aeb 2024-11-26T19:16:57,349 DEBUG [StoreCloser-TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/549a987d77de492d8546f7ef8da171db to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/549a987d77de492d8546f7ef8da171db 2024-11-26T19:16:57,350 DEBUG [StoreCloser-TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/bb50fc14458e4ff493f25015fe5a5e0f to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/bb50fc14458e4ff493f25015fe5a5e0f 2024-11-26T19:16:57,350 DEBUG [StoreCloser-TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/52297031376c4960b8d295fc2485d9c0 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/52297031376c4960b8d295fc2485d9c0 2024-11-26T19:16:57,351 DEBUG [StoreCloser-TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/d9629b526b2840e29e5f097c0c495118 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/d9629b526b2840e29e5f097c0c495118 2024-11-26T19:16:57,352 DEBUG [StoreCloser-TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/bf7e8f9b4a7048d0b9cdfd192ee0f77a to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/bf7e8f9b4a7048d0b9cdfd192ee0f77a 2024-11-26T19:16:57,353 DEBUG [StoreCloser-TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/0ddccb6d4dfc4b54b2c508ce52bf5713 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/0ddccb6d4dfc4b54b2c508ce52bf5713 2024-11-26T19:16:57,353 DEBUG [StoreCloser-TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/ee38a39e45ca41c4ba673e6ea07bbb38 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/ee38a39e45ca41c4ba673e6ea07bbb38 2024-11-26T19:16:57,354 DEBUG [StoreCloser-TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/076c35a352c64be0a3cda5c12aa5731b to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/076c35a352c64be0a3cda5c12aa5731b 2024-11-26T19:16:57,355 DEBUG [StoreCloser-TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/46a2cd1070d14569b1f23ff22d1b6f9d to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/46a2cd1070d14569b1f23ff22d1b6f9d 2024-11-26T19:16:57,355 DEBUG [StoreCloser-TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/2bf74ae587d042c9a08356c0ebca120f to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/2bf74ae587d042c9a08356c0ebca120f 2024-11-26T19:16:57,356 DEBUG [StoreCloser-TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/43e8185d1b654de29a8603bfc9e7f606 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/43e8185d1b654de29a8603bfc9e7f606 2024-11-26T19:16:57,357 DEBUG [StoreCloser-TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/d301166e22d0465eadf5f19317a14cd3 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/d301166e22d0465eadf5f19317a14cd3 2024-11-26T19:16:57,358 DEBUG [StoreCloser-TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/f56d672b42624d72a0a98bb4d648917b to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/f56d672b42624d72a0a98bb4d648917b 2024-11-26T19:16:57,358 DEBUG [StoreCloser-TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/429fd324533341e19f682ec8b8298251 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/429fd324533341e19f682ec8b8298251 2024-11-26T19:16:57,359 DEBUG [StoreCloser-TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/876f2ac06b58413ca60360273c21f9f9 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/876f2ac06b58413ca60360273c21f9f9 2024-11-26T19:16:57,360 DEBUG [StoreCloser-TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/1a802674f0544c2eba1b46d3c0267bfb to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/1a802674f0544c2eba1b46d3c0267bfb 2024-11-26T19:16:57,361 DEBUG [StoreCloser-TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/a6045f6ff2174af3bdfed1277c848412 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/a6045f6ff2174af3bdfed1277c848412 2024-11-26T19:16:57,362 DEBUG [StoreCloser-TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/B/9da0a6ee2caf44cfbf4ffbdea845fc85, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/B/74374299f8d342ef886cce0569e0d824, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/B/a2569dff74ba4043b514760294e8b147, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/B/ebff55f6c85147a8b651e7722a0d7c2c, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/B/0bb0305a8ec2489190b2197d1d62d6d2, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/B/7d89b5162b5b44938a2428c5871d5706, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/B/6100ea3d0fe04142b1e874fbcc533cf3, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/B/9c3cb46b46e244c6bef2e295460b1ab7, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/B/53e57dc86ac544319bee86011f6ff6ca, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/B/cd980fc48f114e9ea2f096da0d4a4d09, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/B/95f3855b9b474384baaeea19ab4290c0, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/B/db4363f29ea240e9920f9452c1c0b996, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/B/33ffa57def274e60a37aeab3a7445f3c, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/B/b4ccec93922647e982907ebd40a6a66a, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/B/033ab683305b46dbb50c21e56c93ac94, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/B/e44547b5e20744f596ab8b11b0bcd794, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/B/d549d8f97b66450ea46484f502fc53e8, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/B/6149ecd1bc264563af5527de7993cc86, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/B/030b5955d4cf4e14b3267911ae5935b7, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/B/af1f3490557443da860fdb0890970875, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/B/cc6d07f7b13244b4b1bed0381883286a, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/B/bf383d07a7e94268ae36e843a347e1f0, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/B/baf08bd11b954ecda254b5aec7d5fae3, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/B/ca886842bcb34290a9412979e3398949, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/B/c37ba3e81fa647b68e17543efdc9bbd7] to archive 2024-11-26T19:16:57,363 DEBUG [StoreCloser-TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-26T19:16:57,364 DEBUG [StoreCloser-TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/B/9da0a6ee2caf44cfbf4ffbdea845fc85 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/B/9da0a6ee2caf44cfbf4ffbdea845fc85 2024-11-26T19:16:57,364 DEBUG [StoreCloser-TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/B/74374299f8d342ef886cce0569e0d824 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/B/74374299f8d342ef886cce0569e0d824 2024-11-26T19:16:57,365 DEBUG [StoreCloser-TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/B/a2569dff74ba4043b514760294e8b147 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/B/a2569dff74ba4043b514760294e8b147 2024-11-26T19:16:57,366 DEBUG [StoreCloser-TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/B/ebff55f6c85147a8b651e7722a0d7c2c to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/B/ebff55f6c85147a8b651e7722a0d7c2c 2024-11-26T19:16:57,367 DEBUG [StoreCloser-TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/B/0bb0305a8ec2489190b2197d1d62d6d2 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/B/0bb0305a8ec2489190b2197d1d62d6d2 2024-11-26T19:16:57,367 DEBUG [StoreCloser-TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/B/7d89b5162b5b44938a2428c5871d5706 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/B/7d89b5162b5b44938a2428c5871d5706 2024-11-26T19:16:57,368 DEBUG [StoreCloser-TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/B/6100ea3d0fe04142b1e874fbcc533cf3 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/B/6100ea3d0fe04142b1e874fbcc533cf3 2024-11-26T19:16:57,369 DEBUG [StoreCloser-TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/B/9c3cb46b46e244c6bef2e295460b1ab7 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/B/9c3cb46b46e244c6bef2e295460b1ab7 2024-11-26T19:16:57,370 DEBUG [StoreCloser-TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/B/53e57dc86ac544319bee86011f6ff6ca to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/B/53e57dc86ac544319bee86011f6ff6ca 2024-11-26T19:16:57,371 DEBUG [StoreCloser-TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/B/cd980fc48f114e9ea2f096da0d4a4d09 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/B/cd980fc48f114e9ea2f096da0d4a4d09 2024-11-26T19:16:57,372 DEBUG [StoreCloser-TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/B/95f3855b9b474384baaeea19ab4290c0 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/B/95f3855b9b474384baaeea19ab4290c0 2024-11-26T19:16:57,372 DEBUG [StoreCloser-TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/B/db4363f29ea240e9920f9452c1c0b996 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/B/db4363f29ea240e9920f9452c1c0b996 2024-11-26T19:16:57,373 DEBUG [StoreCloser-TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/B/33ffa57def274e60a37aeab3a7445f3c to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/B/33ffa57def274e60a37aeab3a7445f3c 2024-11-26T19:16:57,374 DEBUG [StoreCloser-TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/B/b4ccec93922647e982907ebd40a6a66a to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/B/b4ccec93922647e982907ebd40a6a66a 2024-11-26T19:16:57,374 DEBUG [StoreCloser-TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/B/033ab683305b46dbb50c21e56c93ac94 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/B/033ab683305b46dbb50c21e56c93ac94 2024-11-26T19:16:57,375 DEBUG [StoreCloser-TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/B/e44547b5e20744f596ab8b11b0bcd794 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/B/e44547b5e20744f596ab8b11b0bcd794 2024-11-26T19:16:57,376 DEBUG [StoreCloser-TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/B/d549d8f97b66450ea46484f502fc53e8 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/B/d549d8f97b66450ea46484f502fc53e8 2024-11-26T19:16:57,376 DEBUG [StoreCloser-TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/B/6149ecd1bc264563af5527de7993cc86 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/B/6149ecd1bc264563af5527de7993cc86 2024-11-26T19:16:57,377 DEBUG [StoreCloser-TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/B/030b5955d4cf4e14b3267911ae5935b7 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/B/030b5955d4cf4e14b3267911ae5935b7 2024-11-26T19:16:57,378 DEBUG [StoreCloser-TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/B/af1f3490557443da860fdb0890970875 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/B/af1f3490557443da860fdb0890970875 2024-11-26T19:16:57,378 DEBUG [StoreCloser-TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/B/cc6d07f7b13244b4b1bed0381883286a to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/B/cc6d07f7b13244b4b1bed0381883286a 2024-11-26T19:16:57,379 DEBUG [StoreCloser-TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/B/bf383d07a7e94268ae36e843a347e1f0 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/B/bf383d07a7e94268ae36e843a347e1f0 2024-11-26T19:16:57,380 DEBUG [StoreCloser-TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/B/baf08bd11b954ecda254b5aec7d5fae3 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/B/baf08bd11b954ecda254b5aec7d5fae3 2024-11-26T19:16:57,380 DEBUG [StoreCloser-TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/B/ca886842bcb34290a9412979e3398949 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/B/ca886842bcb34290a9412979e3398949 2024-11-26T19:16:57,381 DEBUG [StoreCloser-TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/B/c37ba3e81fa647b68e17543efdc9bbd7 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/B/c37ba3e81fa647b68e17543efdc9bbd7 2024-11-26T19:16:57,382 DEBUG [StoreCloser-TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/C/5c228e0d704a4e819dd191c5b945755c, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/C/0fc42d0edd174075bd86848d968acc8f, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/C/30b32d80285d41659daee1731648bf36, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/C/484760cc712349a5adda6432ccd0f622, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/C/15c04663c1744d8ca2577de7b569df02, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/C/4df303a8950f4039ae03560bb28cbd22, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/C/b519d0bf2c1744aab5fc25030aa9a062, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/C/a1609f64efe4494f9d706ac79e03d3dd, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/C/569f54ee3b00493da947546d9599b245, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/C/c99db25f98c64c4a878c80e1720d6611, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/C/3653f949eea7465cb4954db45b016357, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/C/2de53e2a3cd84482b3e52ec82eda114f, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/C/007764b90f694955baae50481b1155f2, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/C/dbc0a37b13b547108005db1715274e7e, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/C/f45532affc4342e086eb0733c882787b, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/C/f6aa479c4752431b8b11bc35a1dd2c7b, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/C/6112da08b8ca41819ed10ac00fc90ef4, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/C/cb3a0b5aca7e4857b9d4ae30b3ed702e, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/C/e79c2a3985154b9786f5bc8e45188109, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/C/0b1d53e991d9421f8cf493f236f829ec, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/C/8c91ed7d69f249af88fce89f9c831f2f, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/C/3de36818353c47d7a012c902b2451f22, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/C/ea4b28c72c554ac4ab8c7537a21c58d2, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/C/46fd39636f8743bbbfe3e3431461a1c3, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/C/36ee3db87ec148d5ac65a9480b0dc44f] to archive 2024-11-26T19:16:57,383 DEBUG [StoreCloser-TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-26T19:16:57,383 DEBUG [StoreCloser-TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/C/5c228e0d704a4e819dd191c5b945755c to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/C/5c228e0d704a4e819dd191c5b945755c 2024-11-26T19:16:57,384 DEBUG [StoreCloser-TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/C/0fc42d0edd174075bd86848d968acc8f to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/C/0fc42d0edd174075bd86848d968acc8f 2024-11-26T19:16:57,385 DEBUG [StoreCloser-TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/C/30b32d80285d41659daee1731648bf36 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/C/30b32d80285d41659daee1731648bf36 2024-11-26T19:16:57,386 DEBUG [StoreCloser-TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/C/484760cc712349a5adda6432ccd0f622 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/C/484760cc712349a5adda6432ccd0f622 2024-11-26T19:16:57,386 DEBUG [StoreCloser-TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/C/15c04663c1744d8ca2577de7b569df02 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/C/15c04663c1744d8ca2577de7b569df02 2024-11-26T19:16:57,387 DEBUG [StoreCloser-TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/C/4df303a8950f4039ae03560bb28cbd22 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/C/4df303a8950f4039ae03560bb28cbd22 2024-11-26T19:16:57,387 DEBUG [StoreCloser-TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/C/b519d0bf2c1744aab5fc25030aa9a062 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/C/b519d0bf2c1744aab5fc25030aa9a062 2024-11-26T19:16:57,388 DEBUG [StoreCloser-TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/C/a1609f64efe4494f9d706ac79e03d3dd to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/C/a1609f64efe4494f9d706ac79e03d3dd 2024-11-26T19:16:57,389 DEBUG [StoreCloser-TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/C/569f54ee3b00493da947546d9599b245 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/C/569f54ee3b00493da947546d9599b245 2024-11-26T19:16:57,389 DEBUG [StoreCloser-TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/C/c99db25f98c64c4a878c80e1720d6611 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/C/c99db25f98c64c4a878c80e1720d6611 2024-11-26T19:16:57,390 DEBUG [StoreCloser-TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/C/3653f949eea7465cb4954db45b016357 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/C/3653f949eea7465cb4954db45b016357 2024-11-26T19:16:57,391 DEBUG [StoreCloser-TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/C/2de53e2a3cd84482b3e52ec82eda114f to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/C/2de53e2a3cd84482b3e52ec82eda114f 2024-11-26T19:16:57,391 DEBUG [StoreCloser-TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/C/007764b90f694955baae50481b1155f2 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/C/007764b90f694955baae50481b1155f2 2024-11-26T19:16:57,392 DEBUG [StoreCloser-TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/C/dbc0a37b13b547108005db1715274e7e to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/C/dbc0a37b13b547108005db1715274e7e 2024-11-26T19:16:57,393 DEBUG [StoreCloser-TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/C/f45532affc4342e086eb0733c882787b to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/C/f45532affc4342e086eb0733c882787b 2024-11-26T19:16:57,393 DEBUG [StoreCloser-TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/C/f6aa479c4752431b8b11bc35a1dd2c7b to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/C/f6aa479c4752431b8b11bc35a1dd2c7b 2024-11-26T19:16:57,394 DEBUG [StoreCloser-TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/C/6112da08b8ca41819ed10ac00fc90ef4 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/C/6112da08b8ca41819ed10ac00fc90ef4 2024-11-26T19:16:57,395 DEBUG [StoreCloser-TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/C/cb3a0b5aca7e4857b9d4ae30b3ed702e to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/C/cb3a0b5aca7e4857b9d4ae30b3ed702e 2024-11-26T19:16:57,395 DEBUG [StoreCloser-TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/C/e79c2a3985154b9786f5bc8e45188109 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/C/e79c2a3985154b9786f5bc8e45188109 2024-11-26T19:16:57,396 DEBUG [StoreCloser-TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/C/0b1d53e991d9421f8cf493f236f829ec to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/C/0b1d53e991d9421f8cf493f236f829ec 2024-11-26T19:16:57,397 DEBUG [StoreCloser-TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/C/8c91ed7d69f249af88fce89f9c831f2f to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/C/8c91ed7d69f249af88fce89f9c831f2f 2024-11-26T19:16:57,397 DEBUG [StoreCloser-TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/C/3de36818353c47d7a012c902b2451f22 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/C/3de36818353c47d7a012c902b2451f22 2024-11-26T19:16:57,398 DEBUG [StoreCloser-TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/C/ea4b28c72c554ac4ab8c7537a21c58d2 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/C/ea4b28c72c554ac4ab8c7537a21c58d2 2024-11-26T19:16:57,399 DEBUG [StoreCloser-TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/C/46fd39636f8743bbbfe3e3431461a1c3 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/C/46fd39636f8743bbbfe3e3431461a1c3 2024-11-26T19:16:57,399 DEBUG [StoreCloser-TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/C/36ee3db87ec148d5ac65a9480b0dc44f to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/C/36ee3db87ec148d5ac65a9480b0dc44f 2024-11-26T19:16:57,403 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/recovered.edits/496.seqid, newMaxSeqId=496, maxSeqId=4 2024-11-26T19:16:57,403 INFO [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e. 2024-11-26T19:16:57,404 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegion(1676): Region close journal for 66e19d6a9a9885e5b2c786f2e4c45a8e: Waiting for close lock at 1732648615695Running coprocessor pre-close hooks at 1732648615695Disabling compacts and flushes for region at 1732648615695Disabling writes for close at 1732648615696 (+1 ms)Obtaining lock to block concurrent updates at 1732648615696Preparing flush snapshotting stores in 66e19d6a9a9885e5b2c786f2e4c45a8e at 1732648615696Finished memstore snapshotting TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1732648615696Flushing stores of TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e. at 1732648615697 (+1 ms)Flushing 66e19d6a9a9885e5b2c786f2e4c45a8e/A: creating writer at 1732648615697Flushing mob file 66e19d6a9a9885e5b2c786f2e4c45a8e/A: appending metadata at 1732648615700 (+3 ms)Flushing mob file 66e19d6a9a9885e5b2c786f2e4c45a8e/A: closing flushed file at 1732648615700Flushing 66e19d6a9a9885e5b2c786f2e4c45a8e/A: appending metadata at 1732648616108 (+408 ms)Flushing 66e19d6a9a9885e5b2c786f2e4c45a8e/A: closing flushed file at 1732648616108Flushing 66e19d6a9a9885e5b2c786f2e4c45a8e/B: creating writer at 1732648616515 (+407 ms)Flushing 66e19d6a9a9885e5b2c786f2e4c45a8e/B: appending metadata at 1732648616517 (+2 ms)Flushing 66e19d6a9a9885e5b2c786f2e4c45a8e/B: closing flushed file at 1732648616517Flushing 66e19d6a9a9885e5b2c786f2e4c45a8e/C: creating writer at 1732648616924 (+407 ms)Flushing 66e19d6a9a9885e5b2c786f2e4c45a8e/C: appending metadata at 1732648616926 (+2 ms)Flushing 66e19d6a9a9885e5b2c786f2e4c45a8e/C: closing flushed file at 1732648616926Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1a60d5b6: reopening flushed file at 1732648617332 (+406 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2361e748: reopening flushed file at 1732648617335 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1338c0e6: reopening flushed file at 1732648617338 (+3 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 66e19d6a9a9885e5b2c786f2e4c45a8e in 1645ms, sequenceid=493, compaction requested=false at 1732648617341 (+3 ms)Writing region close event to WAL at 1732648617400 (+59 ms)Running coprocessor post-close hooks at 1732648617403 (+3 ms)Closed at 1732648617403 2024-11-26T19:16:57,405 INFO [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] handler.UnassignRegionHandler(157): Closed 66e19d6a9a9885e5b2c786f2e4c45a8e 2024-11-26T19:16:57,405 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=162 updating hbase:meta row=66e19d6a9a9885e5b2c786f2e4c45a8e, regionState=CLOSED 2024-11-26T19:16:57,407 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=163, ppid=162, state=RUNNABLE, hasLock=false; CloseRegionProcedure 66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 because future has completed 2024-11-26T19:16:57,409 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-11-26T19:16:57,409 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; CloseRegionProcedure 66e19d6a9a9885e5b2c786f2e4c45a8e, server=fe76acf732b2,42785,1732648423970 in 1.8640 sec 2024-11-26T19:16:57,410 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=162, resume processing ppid=161 2024-11-26T19:16:57,410 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, ppid=161, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=66e19d6a9a9885e5b2c786f2e4c45a8e, UNASSIGN in 1.8680 sec 2024-11-26T19:16:57,412 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-11-26T19:16:57,412 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8710 sec 2024-11-26T19:16:57,413 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732648617413"}]},"ts":"1732648617413"} 2024-11-26T19:16:57,414 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-26T19:16:57,414 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-26T19:16:57,416 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8800 sec 2024-11-26T19:16:57,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-26T19:16:57,682 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-26T19:16:57,683 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-26T19:16:57,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] procedure2.ProcedureExecutor(1139): Stored pid=164, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T19:16:57,684 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=164, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T19:16:57,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-26T19:16:57,685 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=164, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T19:16:57,687 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e 2024-11-26T19:16:57,688 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A, FileablePath, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/B, FileablePath, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/C, FileablePath, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/recovered.edits] 2024-11-26T19:16:57,690 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/042390db0c76490892d41a81067a0f68 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/042390db0c76490892d41a81067a0f68 2024-11-26T19:16:57,691 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/5afaa5fdaaa645fca95b64fd46693212 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/A/5afaa5fdaaa645fca95b64fd46693212 2024-11-26T19:16:57,693 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/B/4f4c59ff67114bb992ce7ee308c008e8 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/B/4f4c59ff67114bb992ce7ee308c008e8 2024-11-26T19:16:57,694 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/B/99778df0ed484c2687699f40e64372db to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/B/99778df0ed484c2687699f40e64372db 2024-11-26T19:16:57,695 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/C/4f4e0cb76c9744ceae642d574c1ab9d6 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/C/4f4e0cb76c9744ceae642d574c1ab9d6 2024-11-26T19:16:57,696 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/C/c06ced022c4b496fac6fdb7e415b60e5 to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/C/c06ced022c4b496fac6fdb7e415b60e5 2024-11-26T19:16:57,698 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/recovered.edits/496.seqid to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e/recovered.edits/496.seqid 2024-11-26T19:16:57,698 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/default/TestAcidGuarantees/66e19d6a9a9885e5b2c786f2e4c45a8e 2024-11-26T19:16:57,698 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-26T19:16:57,698 DEBUG [PEWorker-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-26T19:16:57,699 DEBUG [PEWorker-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-26T19:16:57,701 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411260454914cdd1241479e8214a573632974_66e19d6a9a9885e5b2c786f2e4c45a8e to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411260454914cdd1241479e8214a573632974_66e19d6a9a9885e5b2c786f2e4c45a8e 2024-11-26T19:16:57,702 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411260615fc48ba0d4965b996a5c8480cd3f7_66e19d6a9a9885e5b2c786f2e4c45a8e to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411260615fc48ba0d4965b996a5c8480cd3f7_66e19d6a9a9885e5b2c786f2e4c45a8e 2024-11-26T19:16:57,702 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126197a014aed294ca6a7c1648a090a3884_66e19d6a9a9885e5b2c786f2e4c45a8e to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126197a014aed294ca6a7c1648a090a3884_66e19d6a9a9885e5b2c786f2e4c45a8e 2024-11-26T19:16:57,703 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112619b586e2bfb84315a264b2d755b47639_66e19d6a9a9885e5b2c786f2e4c45a8e to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112619b586e2bfb84315a264b2d755b47639_66e19d6a9a9885e5b2c786f2e4c45a8e 2024-11-26T19:16:57,704 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411261e67226e811a4483a6b73cdc2fe0c646_66e19d6a9a9885e5b2c786f2e4c45a8e to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411261e67226e811a4483a6b73cdc2fe0c646_66e19d6a9a9885e5b2c786f2e4c45a8e 2024-11-26T19:16:57,705 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411263446671321104df8bcb1f58b7f1f962f_66e19d6a9a9885e5b2c786f2e4c45a8e to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411263446671321104df8bcb1f58b7f1f962f_66e19d6a9a9885e5b2c786f2e4c45a8e 2024-11-26T19:16:57,705 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411263b43cabc3a724edda5eef4e6eb058372_66e19d6a9a9885e5b2c786f2e4c45a8e to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411263b43cabc3a724edda5eef4e6eb058372_66e19d6a9a9885e5b2c786f2e4c45a8e 2024-11-26T19:16:57,706 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411264946c4b3db4a4e0ba9497bde1d48b055_66e19d6a9a9885e5b2c786f2e4c45a8e to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411264946c4b3db4a4e0ba9497bde1d48b055_66e19d6a9a9885e5b2c786f2e4c45a8e 2024-11-26T19:16:57,707 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112652ada8e4a3bd43e5b12a620372f870a6_66e19d6a9a9885e5b2c786f2e4c45a8e to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112652ada8e4a3bd43e5b12a620372f870a6_66e19d6a9a9885e5b2c786f2e4c45a8e 2024-11-26T19:16:57,708 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112653d591040fd84deb93623e97b63da51f_66e19d6a9a9885e5b2c786f2e4c45a8e to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112653d591040fd84deb93623e97b63da51f_66e19d6a9a9885e5b2c786f2e4c45a8e 2024-11-26T19:16:57,709 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112667a01ddf085642a090509436bfd920c6_66e19d6a9a9885e5b2c786f2e4c45a8e to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112667a01ddf085642a090509436bfd920c6_66e19d6a9a9885e5b2c786f2e4c45a8e 2024-11-26T19:16:57,709 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411266d64842c8a034640bbe8dfa78db60bcb_66e19d6a9a9885e5b2c786f2e4c45a8e to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411266d64842c8a034640bbe8dfa78db60bcb_66e19d6a9a9885e5b2c786f2e4c45a8e 2024-11-26T19:16:57,710 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411269251111d12fb4f379dddb3a2caa08920_66e19d6a9a9885e5b2c786f2e4c45a8e to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411269251111d12fb4f379dddb3a2caa08920_66e19d6a9a9885e5b2c786f2e4c45a8e 2024-11-26T19:16:57,711 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112696bda8669b7e4edbb0c47ae206e28f9b_66e19d6a9a9885e5b2c786f2e4c45a8e to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112696bda8669b7e4edbb0c47ae206e28f9b_66e19d6a9a9885e5b2c786f2e4c45a8e 2024-11-26T19:16:57,712 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126c7566238493045509a38f9b11a367c2b_66e19d6a9a9885e5b2c786f2e4c45a8e to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126c7566238493045509a38f9b11a367c2b_66e19d6a9a9885e5b2c786f2e4c45a8e 2024-11-26T19:16:57,712 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126c8cc880d47a3449797351f44efc2fd1c_66e19d6a9a9885e5b2c786f2e4c45a8e to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126c8cc880d47a3449797351f44efc2fd1c_66e19d6a9a9885e5b2c786f2e4c45a8e 2024-11-26T19:16:57,713 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126ca2b373b22254a5f8dd790763ef52acf_66e19d6a9a9885e5b2c786f2e4c45a8e to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126ca2b373b22254a5f8dd790763ef52acf_66e19d6a9a9885e5b2c786f2e4c45a8e 2024-11-26T19:16:57,714 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126daaf468e98b847adbe393ef951affcba_66e19d6a9a9885e5b2c786f2e4c45a8e to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126daaf468e98b847adbe393ef951affcba_66e19d6a9a9885e5b2c786f2e4c45a8e 2024-11-26T19:16:57,715 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126e9cb0cd629fb43b69cca0526eb5b1f52_66e19d6a9a9885e5b2c786f2e4c45a8e to hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126e9cb0cd629fb43b69cca0526eb5b1f52_66e19d6a9a9885e5b2c786f2e4c45a8e 2024-11-26T19:16:57,716 DEBUG [PEWorker-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-26T19:16:57,718 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=164, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T19:16:57,720 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-26T19:16:57,722 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-26T19:16:57,723 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=164, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T19:16:57,723 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-26T19:16:57,723 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732648617723"}]},"ts":"9223372036854775807"} 2024-11-26T19:16:57,725 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-26T19:16:57,725 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 66e19d6a9a9885e5b2c786f2e4c45a8e, NAME => 'TestAcidGuarantees,,1732648582353.66e19d6a9a9885e5b2c786f2e4c45a8e.', STARTKEY => '', ENDKEY => ''}] 2024-11-26T19:16:57,725 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-26T19:16:57,725 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732648617725"}]},"ts":"9223372036854775807"} 2024-11-26T19:16:57,727 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-26T19:16:57,727 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=164, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T19:16:57,729 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 44 msec 2024-11-26T19:16:57,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-26T19:16:57,792 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-26T19:16:57,792 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-26T19:16:57,803 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=247 (was 246) - Thread LEAK? -, OpenFileDescriptor=449 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=277 (was 307), ProcessCount=11 (was 11), AvailableMemoryMB=5505 (was 5694) 2024-11-26T19:16:57,803 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-26T19:16:57,803 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-26T19:16:57,803 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-26T19:16:57,803 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:16:57,803 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:16:57,803 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T19:16:57,804 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-26T19:16:57,804 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=694036216, stopped=false 2024-11-26T19:16:57,804 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=fe76acf732b2,33811,1732648423099 2024-11-26T19:16:57,806 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42785-0x10155b2b5d30001, quorum=127.0.0.1:62039, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-26T19:16:57,806 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33811-0x10155b2b5d30000, quorum=127.0.0.1:62039, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-26T19:16:57,806 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33811-0x10155b2b5d30000, quorum=127.0.0.1:62039, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T19:16:57,806 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42785-0x10155b2b5d30001, quorum=127.0.0.1:62039, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T19:16:57,806 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T19:16:57,807 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:33811-0x10155b2b5d30000, quorum=127.0.0.1:62039, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-26T19:16:57,807 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:42785-0x10155b2b5d30001, quorum=127.0.0.1:62039, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-26T19:16:57,807 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-26T19:16:57,807 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-26T19:16:57,807 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:16:57,807 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'fe76acf732b2,42785,1732648423970' ***** 2024-11-26T19:16:57,807 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-26T19:16:57,808 INFO [RS:0;fe76acf732b2:42785 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-26T19:16:57,808 INFO [RS:0;fe76acf732b2:42785 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-26T19:16:57,808 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-26T19:16:57,808 INFO [RS:0;fe76acf732b2:42785 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-26T19:16:57,808 INFO [RS:0;fe76acf732b2:42785 {}] regionserver.HRegionServer(959): stopping server fe76acf732b2,42785,1732648423970 2024-11-26T19:16:57,808 INFO [RS:0;fe76acf732b2:42785 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-26T19:16:57,808 INFO [RS:0;fe76acf732b2:42785 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;fe76acf732b2:42785. 2024-11-26T19:16:57,808 DEBUG [RS:0;fe76acf732b2:42785 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-26T19:16:57,809 DEBUG [RS:0;fe76acf732b2:42785 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:16:57,809 INFO [RS:0;fe76acf732b2:42785 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-26T19:16:57,809 INFO [RS:0;fe76acf732b2:42785 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-26T19:16:57,809 INFO [RS:0;fe76acf732b2:42785 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-26T19:16:57,809 INFO [RS:0;fe76acf732b2:42785 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-26T19:16:57,809 INFO [RS:0;fe76acf732b2:42785 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-26T19:16:57,809 DEBUG [RS:0;fe76acf732b2:42785 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-26T19:16:57,810 DEBUG [RS:0;fe76acf732b2:42785 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-26T19:16:57,810 DEBUG [RS_CLOSE_META-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-26T19:16:57,810 INFO [RS_CLOSE_META-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-26T19:16:57,810 DEBUG [RS_CLOSE_META-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-26T19:16:57,810 DEBUG [RS_CLOSE_META-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-26T19:16:57,810 DEBUG [RS_CLOSE_META-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-26T19:16:57,810 INFO [RS_CLOSE_META-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-26T19:16:57,814 INFO [regionserver/fe76acf732b2:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-26T19:16:57,834 DEBUG [RS_CLOSE_META-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/hbase/meta/1588230740/.tmp/info/8a474c54598b4ffea3a0573fdec00084 is 95, key is TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596./info:/1732648456440/DeleteFamily/seqid=0 2024-11-26T19:16:57,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742446_1622 (size=6514) 2024-11-26T19:16:58,010 DEBUG [RS:0;fe76acf732b2:42785 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-26T19:16:58,210 DEBUG [RS:0;fe76acf732b2:42785 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-26T19:16:58,237 INFO [RS_CLOSE_META-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/hbase/meta/1588230740/.tmp/info/8a474c54598b4ffea3a0573fdec00084 2024-11-26T19:16:58,255 DEBUG [RS_CLOSE_META-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/hbase/meta/1588230740/.tmp/ns/00fe05ce30c74d5dbdbd44080c2e1838 is 93, key is TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596./ns:/1732648456396/DeleteFamily/seqid=0 2024-11-26T19:16:58,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742447_1623 (size=5906) 2024-11-26T19:16:58,410 DEBUG [RS:0;fe76acf732b2:42785 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-26T19:16:58,426 INFO [regionserver/fe76acf732b2:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-26T19:16:58,426 INFO [regionserver/fe76acf732b2:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-26T19:16:58,610 DEBUG [RS:0;fe76acf732b2:42785 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-26T19:16:58,659 INFO [RS_CLOSE_META-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/hbase/meta/1588230740/.tmp/ns/00fe05ce30c74d5dbdbd44080c2e1838 2024-11-26T19:16:58,676 DEBUG [RS_CLOSE_META-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/hbase/meta/1588230740/.tmp/rep_barrier/2920031576db4b21ac5b43debe0037a3 is 102, key is TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596./rep_barrier:/1732648456396/DeleteFamily/seqid=0 2024-11-26T19:16:58,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742448_1624 (size=6025) 2024-11-26T19:16:58,811 DEBUG [RS:0;fe76acf732b2:42785 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-26T19:16:59,011 DEBUG [RS:0;fe76acf732b2:42785 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-26T19:16:59,079 INFO [RS_CLOSE_META-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/hbase/meta/1588230740/.tmp/rep_barrier/2920031576db4b21ac5b43debe0037a3 2024-11-26T19:16:59,097 DEBUG [RS_CLOSE_META-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/hbase/meta/1588230740/.tmp/table/9a0d4eacc7c944f5a9d0268aaeab0b45 is 96, key is TestAcidGuarantees,,1732648428071.76c4ff0c848e5ffb5ad9989537063596./table:/1732648456396/DeleteFamily/seqid=0 2024-11-26T19:16:59,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742449_1625 (size=5936) 2024-11-26T19:16:59,211 DEBUG [RS:0;fe76acf732b2:42785 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-26T19:16:59,411 DEBUG [RS:0;fe76acf732b2:42785 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-26T19:16:59,501 INFO [RS_CLOSE_META-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/hbase/meta/1588230740/.tmp/table/9a0d4eacc7c944f5a9d0268aaeab0b45 2024-11-26T19:16:59,504 DEBUG [RS_CLOSE_META-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/hbase/meta/1588230740/.tmp/info/8a474c54598b4ffea3a0573fdec00084 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/hbase/meta/1588230740/info/8a474c54598b4ffea3a0573fdec00084 2024-11-26T19:16:59,507 INFO [RS_CLOSE_META-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/hbase/meta/1588230740/info/8a474c54598b4ffea3a0573fdec00084, entries=12, sequenceid=90, filesize=6.4 K 2024-11-26T19:16:59,507 DEBUG [RS_CLOSE_META-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/hbase/meta/1588230740/.tmp/ns/00fe05ce30c74d5dbdbd44080c2e1838 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/hbase/meta/1588230740/ns/00fe05ce30c74d5dbdbd44080c2e1838 2024-11-26T19:16:59,510 INFO [RS_CLOSE_META-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/hbase/meta/1588230740/ns/00fe05ce30c74d5dbdbd44080c2e1838, entries=8, sequenceid=90, filesize=5.8 K 2024-11-26T19:16:59,510 DEBUG [RS_CLOSE_META-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/hbase/meta/1588230740/.tmp/rep_barrier/2920031576db4b21ac5b43debe0037a3 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/hbase/meta/1588230740/rep_barrier/2920031576db4b21ac5b43debe0037a3 2024-11-26T19:16:59,513 INFO [RS_CLOSE_META-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/hbase/meta/1588230740/rep_barrier/2920031576db4b21ac5b43debe0037a3, entries=6, sequenceid=90, filesize=5.9 K 2024-11-26T19:16:59,513 DEBUG [RS_CLOSE_META-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/hbase/meta/1588230740/.tmp/table/9a0d4eacc7c944f5a9d0268aaeab0b45 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/hbase/meta/1588230740/table/9a0d4eacc7c944f5a9d0268aaeab0b45 2024-11-26T19:16:59,516 INFO [RS_CLOSE_META-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/hbase/meta/1588230740/table/9a0d4eacc7c944f5a9d0268aaeab0b45, entries=7, sequenceid=90, filesize=5.8 K 2024-11-26T19:16:59,516 INFO [RS_CLOSE_META-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1706ms, sequenceid=90, compaction requested=false 2024-11-26T19:16:59,521 DEBUG [RS_CLOSE_META-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-26T19:16:59,521 DEBUG [RS_CLOSE_META-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-26T19:16:59,521 INFO [RS_CLOSE_META-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-26T19:16:59,521 DEBUG [RS_CLOSE_META-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732648617809Running coprocessor pre-close hooks at 1732648617810 (+1 ms)Disabling compacts and flushes for region at 1732648617810Disabling writes for close at 1732648617810Obtaining lock to block concurrent updates at 1732648617810Preparing flush snapshotting stores in 1588230740 at 1732648617810Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1732648617810Flushing stores of hbase:meta,,1.1588230740 at 1732648617811 (+1 ms)Flushing 1588230740/info: creating writer at 1732648617811Flushing 1588230740/info: appending metadata at 1732648617833 (+22 ms)Flushing 1588230740/info: closing flushed file at 1732648617833Flushing 1588230740/ns: creating writer at 1732648618241 (+408 ms)Flushing 1588230740/ns: appending metadata at 1732648618255 (+14 ms)Flushing 1588230740/ns: closing flushed file at 1732648618255Flushing 1588230740/rep_barrier: creating writer at 1732648618662 (+407 ms)Flushing 1588230740/rep_barrier: appending metadata at 1732648618676 (+14 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1732648618676Flushing 1588230740/table: creating writer at 1732648619082 (+406 ms)Flushing 1588230740/table: appending metadata at 1732648619097 (+15 ms)Flushing 1588230740/table: closing flushed file at 1732648619097Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@607c69b5: reopening flushed file at 1732648619504 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7400ebf9: reopening flushed file at 1732648619507 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7dae709b: reopening flushed file at 1732648619510 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@9b7caef: reopening flushed file at 1732648619513 (+3 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1706ms, sequenceid=90, compaction requested=false at 1732648619516 (+3 ms)Writing region close event to WAL at 1732648619519 (+3 ms)Running coprocessor post-close hooks at 1732648619521 (+2 ms)Closed at 1732648619521 2024-11-26T19:16:59,522 DEBUG [RS_CLOSE_META-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-26T19:16:59,612 INFO [RS:0;fe76acf732b2:42785 {}] regionserver.HRegionServer(976): stopping server fe76acf732b2,42785,1732648423970; all regions closed. 2024-11-26T19:16:59,616 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741834_1010 (size=25344) 2024-11-26T19:16:59,618 DEBUG [RS:0;fe76acf732b2:42785 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/oldWALs 2024-11-26T19:16:59,618 INFO [RS:0;fe76acf732b2:42785 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL fe76acf732b2%2C42785%2C1732648423970.meta:.meta(num 1732648427581) 2024-11-26T19:16:59,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741832_1008 (size=16919571) 2024-11-26T19:16:59,621 DEBUG [RS:0;fe76acf732b2:42785 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/oldWALs 2024-11-26T19:16:59,621 INFO [RS:0;fe76acf732b2:42785 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL fe76acf732b2%2C42785%2C1732648423970:(num 1732648426587) 2024-11-26T19:16:59,621 DEBUG [RS:0;fe76acf732b2:42785 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:16:59,621 INFO [RS:0;fe76acf732b2:42785 {}] regionserver.LeaseManager(133): Closed leases 2024-11-26T19:16:59,621 INFO [RS:0;fe76acf732b2:42785 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-26T19:16:59,622 INFO [RS:0;fe76acf732b2:42785 {}] hbase.ChoreService(370): Chore service for: regionserver/fe76acf732b2:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-26T19:16:59,622 INFO [RS:0;fe76acf732b2:42785 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-26T19:16:59,622 INFO [regionserver/fe76acf732b2:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-26T19:16:59,623 INFO [RS:0;fe76acf732b2:42785 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:42785 2024-11-26T19:16:59,626 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42785-0x10155b2b5d30001, quorum=127.0.0.1:62039, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/fe76acf732b2,42785,1732648423970 2024-11-26T19:16:59,626 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33811-0x10155b2b5d30000, quorum=127.0.0.1:62039, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-26T19:16:59,626 INFO [RS:0;fe76acf732b2:42785 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-26T19:16:59,627 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [fe76acf732b2,42785,1732648423970] 2024-11-26T19:16:59,631 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/fe76acf732b2,42785,1732648423970 already deleted, retry=false 2024-11-26T19:16:59,631 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; fe76acf732b2,42785,1732648423970 expired; onlineServers=0 2024-11-26T19:16:59,631 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'fe76acf732b2,33811,1732648423099' ***** 2024-11-26T19:16:59,631 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-26T19:16:59,631 INFO [M:0;fe76acf732b2:33811 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-26T19:16:59,631 INFO [M:0;fe76acf732b2:33811 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-26T19:16:59,631 DEBUG [M:0;fe76acf732b2:33811 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-26T19:16:59,631 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-26T19:16:59,631 DEBUG [M:0;fe76acf732b2:33811 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-26T19:16:59,631 DEBUG [master/fe76acf732b2:0:becomeActiveMaster-HFileCleaner.large.0-1732648426310 {}] cleaner.HFileCleaner(306): Exit Thread[master/fe76acf732b2:0:becomeActiveMaster-HFileCleaner.large.0-1732648426310,5,FailOnTimeoutGroup] 2024-11-26T19:16:59,631 DEBUG [master/fe76acf732b2:0:becomeActiveMaster-HFileCleaner.small.0-1732648426311 {}] cleaner.HFileCleaner(306): Exit Thread[master/fe76acf732b2:0:becomeActiveMaster-HFileCleaner.small.0-1732648426311,5,FailOnTimeoutGroup] 2024-11-26T19:16:59,631 INFO [M:0;fe76acf732b2:33811 {}] hbase.ChoreService(370): Chore service for: master/fe76acf732b2:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-26T19:16:59,632 INFO [M:0;fe76acf732b2:33811 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-26T19:16:59,632 DEBUG [M:0;fe76acf732b2:33811 {}] master.HMaster(1795): Stopping service threads 2024-11-26T19:16:59,632 INFO [M:0;fe76acf732b2:33811 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-26T19:16:59,632 INFO [M:0;fe76acf732b2:33811 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T19:16:59,632 ERROR [M:0;fe76acf732b2:33811 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[IPC Client (879580466) connection to localhost/127.0.0.1:43653 from jenkins,5,PEWorkerGroup] Thread[IPC Parameter Sending Thread for localhost/127.0.0.1:43653,5,PEWorkerGroup] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-26T19:16:59,633 INFO [M:0;fe76acf732b2:33811 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-26T19:16:59,633 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-26T19:16:59,634 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33811-0x10155b2b5d30000, quorum=127.0.0.1:62039, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-26T19:16:59,634 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33811-0x10155b2b5d30000, quorum=127.0.0.1:62039, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T19:16:59,634 DEBUG [M:0;fe76acf732b2:33811 {}] zookeeper.ZKUtil(347): master:33811-0x10155b2b5d30000, quorum=127.0.0.1:62039, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-26T19:16:59,634 WARN [M:0;fe76acf732b2:33811 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-26T19:16:59,634 INFO [M:0;fe76acf732b2:33811 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/.lastflushedseqids 2024-11-26T19:16:59,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742450_1626 (size=72) 2024-11-26T19:16:59,729 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42785-0x10155b2b5d30001, quorum=127.0.0.1:62039, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T19:16:59,729 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42785-0x10155b2b5d30001, quorum=127.0.0.1:62039, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T19:16:59,729 INFO [RS:0;fe76acf732b2:42785 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-26T19:16:59,730 INFO [RS:0;fe76acf732b2:42785 {}] regionserver.HRegionServer(1031): Exiting; stopping=fe76acf732b2,42785,1732648423970; zookeeper connection closed. 2024-11-26T19:16:59,730 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@4a06b75 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@4a06b75 2024-11-26T19:16:59,730 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-26T19:17:00,044 INFO [M:0;fe76acf732b2:33811 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-26T19:17:00,045 INFO [M:0;fe76acf732b2:33811 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-26T19:17:00,045 DEBUG [M:0;fe76acf732b2:33811 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-26T19:17:00,045 INFO [M:0;fe76acf732b2:33811 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T19:17:00,045 DEBUG [M:0;fe76acf732b2:33811 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T19:17:00,045 DEBUG [M:0;fe76acf732b2:33811 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-26T19:17:00,045 DEBUG [M:0;fe76acf732b2:33811 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T19:17:00,045 INFO [M:0;fe76acf732b2:33811 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=771.06 KB heapSize=944.44 KB 2024-11-26T19:17:00,060 DEBUG [M:0;fe76acf732b2:33811 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/a6e597e396cf4be5ad9aba45966ad2d0 is 82, key is hbase:meta,,1/info:regioninfo/1732648427673/Put/seqid=0 2024-11-26T19:17:00,063 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742451_1627 (size=5672) 2024-11-26T19:17:00,463 INFO [M:0;fe76acf732b2:33811 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2145 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/a6e597e396cf4be5ad9aba45966ad2d0 2024-11-26T19:17:00,483 DEBUG [M:0;fe76acf732b2:33811 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/522083e92abe4e6099416198debb929d is 2481, key is \x00\x00\x00\x00\x00\x00\x00\x8D/proc:d/1732648584416/Put/seqid=0 2024-11-26T19:17:00,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742452_1628 (size=44380) 2024-11-26T19:17:00,887 INFO [M:0;fe76acf732b2:33811 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=770.50 KB at sequenceid=2145 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/522083e92abe4e6099416198debb929d 2024-11-26T19:17:00,890 INFO [M:0;fe76acf732b2:33811 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 522083e92abe4e6099416198debb929d 2024-11-26T19:17:00,904 DEBUG [M:0;fe76acf732b2:33811 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/b13d65af8dd341d590150fec85c0911f is 69, key is fe76acf732b2,42785,1732648423970/rs:state/1732648426328/Put/seqid=0 2024-11-26T19:17:00,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073742453_1629 (size=5156) 2024-11-26T19:17:01,308 INFO [M:0;fe76acf732b2:33811 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2145 (bloomFilter=true), to=hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/b13d65af8dd341d590150fec85c0911f 2024-11-26T19:17:01,311 DEBUG [M:0;fe76acf732b2:33811 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/a6e597e396cf4be5ad9aba45966ad2d0 as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/a6e597e396cf4be5ad9aba45966ad2d0 2024-11-26T19:17:01,314 INFO [M:0;fe76acf732b2:33811 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/a6e597e396cf4be5ad9aba45966ad2d0, entries=8, sequenceid=2145, filesize=5.5 K 2024-11-26T19:17:01,314 DEBUG [M:0;fe76acf732b2:33811 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/522083e92abe4e6099416198debb929d as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/522083e92abe4e6099416198debb929d 2024-11-26T19:17:01,317 INFO [M:0;fe76acf732b2:33811 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 522083e92abe4e6099416198debb929d 2024-11-26T19:17:01,317 INFO [M:0;fe76acf732b2:33811 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/522083e92abe4e6099416198debb929d, entries=164, sequenceid=2145, filesize=43.3 K 2024-11-26T19:17:01,318 DEBUG [M:0;fe76acf732b2:33811 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/b13d65af8dd341d590150fec85c0911f as hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/b13d65af8dd341d590150fec85c0911f 2024-11-26T19:17:01,321 INFO [M:0;fe76acf732b2:33811 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43653/user/jenkins/test-data/98b0d80c-dca3-3f35-0e42-29422766da16/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/b13d65af8dd341d590150fec85c0911f, entries=1, sequenceid=2145, filesize=5.0 K 2024-11-26T19:17:01,321 INFO [M:0;fe76acf732b2:33811 {}] regionserver.HRegion(3140): Finished flush of dataSize ~771.06 KB/789565, heapSize ~944.14 KB/966800, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1276ms, sequenceid=2145, compaction requested=false 2024-11-26T19:17:01,323 INFO [M:0;fe76acf732b2:33811 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T19:17:01,323 DEBUG [M:0;fe76acf732b2:33811 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732648620045Disabling compacts and flushes for region at 1732648620045Disabling writes for close at 1732648620045Obtaining lock to block concurrent updates at 1732648620045Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732648620045Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=789565, getHeapSize=967040, getOffHeapSize=0, getCellsCount=2334 at 1732648620045Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732648620046 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732648620046Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732648620060 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732648620060Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732648620467 (+407 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732648620483 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732648620483Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732648620890 (+407 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732648620904 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732648620904Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@745d8fc6: reopening flushed file at 1732648621311 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@61756ad7: reopening flushed file at 1732648621314 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6c62a529: reopening flushed file at 1732648621317 (+3 ms)Finished flush of dataSize ~771.06 KB/789565, heapSize ~944.14 KB/966800, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1276ms, sequenceid=2145, compaction requested=false at 1732648621321 (+4 ms)Writing region close event to WAL at 1732648621322 (+1 ms)Closed at 1732648621322 2024-11-26T19:17:01,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34683 is added to blk_1073741830_1006 (size=930165) 2024-11-26T19:17:01,326 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-26T19:17:01,326 INFO [M:0;fe76acf732b2:33811 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-26T19:17:01,326 INFO [M:0;fe76acf732b2:33811 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:33811 2024-11-26T19:17:01,326 INFO [M:0;fe76acf732b2:33811 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-26T19:17:01,428 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33811-0x10155b2b5d30000, quorum=127.0.0.1:62039, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T19:17:01,428 INFO [M:0;fe76acf732b2:33811 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-26T19:17:01,428 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33811-0x10155b2b5d30000, quorum=127.0.0.1:62039, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T19:17:01,432 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@292d50c4{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-26T19:17:01,435 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@667c6eba{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-26T19:17:01,435 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-26T19:17:01,435 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@72782cb0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-26T19:17:01,435 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@54a7b62e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/efb451b3-ba8c-40d2-3b0c-db6f88deea36/hadoop.log.dir/,STOPPED} 2024-11-26T19:17:01,438 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-26T19:17:01,438 WARN [BP-777500891-172.17.0.2-1732648419325 heartbeating to localhost/127.0.0.1:43653 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-26T19:17:01,438 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-26T19:17:01,438 WARN [BP-777500891-172.17.0.2-1732648419325 heartbeating to localhost/127.0.0.1:43653 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-777500891-172.17.0.2-1732648419325 (Datanode Uuid 4ad75c6e-6c93-4de8-8725-1def329addcb) service to localhost/127.0.0.1:43653 2024-11-26T19:17:01,440 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/efb451b3-ba8c-40d2-3b0c-db6f88deea36/cluster_7d054ae1-c0e9-960d-2aea-ddd34a033944/data/data1/current/BP-777500891-172.17.0.2-1732648419325 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-26T19:17:01,440 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/efb451b3-ba8c-40d2-3b0c-db6f88deea36/cluster_7d054ae1-c0e9-960d-2aea-ddd34a033944/data/data2/current/BP-777500891-172.17.0.2-1732648419325 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-26T19:17:01,441 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-26T19:17:01,448 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@2341a969{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-26T19:17:01,449 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@2d064751{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-26T19:17:01,449 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-26T19:17:01,449 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@130d168e{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-26T19:17:01,449 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1a58fd8a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/efb451b3-ba8c-40d2-3b0c-db6f88deea36/hadoop.log.dir/,STOPPED} 2024-11-26T19:17:01,465 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-26T19:17:01,589 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down